From 327f95ec8016616becf6b1584ca1a19e12dbd0c9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:02:42 +0200 Subject: [PATCH 01/15] feat(schema): product_events replaces web_events; identity on session_events; identity_links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 0016 + Tinybird datasources/MVs. product_events is dual-fed (session_events MV for browser rows, direct ingest for server/mobile) and carries Source + VisitorId/UserId/GroupId; 365-day TTL. identity_links stitches visitor→user pairs out of session_replays. --- apps/cli/src/server/schema/local-inserts.json | 2 +- apps/cli/src/server/schema/local-schema.sql | 102 ++++++--- apps/ingest/src/clickhouse_insert_mappings.rs | 18 +- docs/product-events-funnels.md | 210 ++++++++++++++++++ packages/domain/src/clickhouse/backfill.ts | 1 + .../migrations/0016_product_events.ts | 168 ++++++++++++++ .../src/clickhouse/migrations/index.test.ts | 18 +- .../domain/src/clickhouse/migrations/index.ts | 2 + packages/domain/src/clickhouse/qualify.ts | 1 + .../domain/src/generated/clickhouse-schema.ts | 10 +- .../generated/tinybird-project-manifest.ts | 34 ++- packages/domain/src/tinybird/datasources.ts | 159 +++++++++---- .../domain/src/tinybird/materializations.ts | 55 ++++- .../src/tinybird/retention-matrix.test.ts | 7 +- packages/query-engine/src/ch/tables.ts | 50 ++++- .../generate-clickhouse-insert-mappings.ts | 1 + 16 files changed, 708 insertions(+), 130 deletions(-) create mode 100644 docs/product-events-funnels.md create mode 100644 packages/domain/src/clickhouse/migrations/0016_product_events.ts diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index dbb699904..7f9ae745b 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7", + "projectRevision": "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index d47684e8e..38c6a1735 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7 +-- projectRevision: bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04 -- localSchemaVersion: 5 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -133,6 +133,17 @@ PARTITION BY toDate(Timestamp) ORDER BY (OrgId, ServiceName, Timestamp) TTL Timestamp + INTERVAL 90 DAY; +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen DateTime64(9) +) +ENGINE = ReplacingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + CREATE TABLE IF NOT EXISTS logs ( OrgId LowCardinality(String), Timestamp DateTime64(9), @@ -332,6 +343,30 @@ PARTITION BY toDate(TimeUnix) ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) TTL toDate(TimeUnix) + INTERVAL 90 DAY; +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( OrgId LowCardinality(String), Hour DateTime, @@ -636,6 +671,9 @@ CREATE TABLE IF NOT EXISTS session_events ( NetDurationMs UInt32 DEFAULT 0, ErrorStack String DEFAULT '', Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', INDEX idx_type Type TYPE set(16) GRANULARITY 4 ) ENGINE = MergeTree @@ -833,24 +871,6 @@ PARTITION BY toDate(Hour) ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) TTL toDate(Hour) + INTERVAL 365 DAY; -CREATE TABLE IF NOT EXISTS web_events ( - OrgId LowCardinality(String), - Timestamp DateTime64(9), - SessionId String, - Seq UInt32, - Kind LowCardinality(String), - EventName String, - Host LowCardinality(String), - PagePath String, - Url String, - Attributes Map(String, String), - INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 -) -ENGINE = MergeTree -PARTITION BY toDate(Timestamp) -ORDER BY (OrgId, Timestamp, SessionId, Seq) -TTL toDate(Timestamp) + INTERVAL 30 DAY; - CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS WITH arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, @@ -1063,6 +1083,15 @@ SELECT FROM traces WHERE StatusCode = 'Error'; +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS SELECT OrgId, @@ -1192,6 +1221,26 @@ SELECT FROM metrics_sum GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); + CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS SELECT OrgId, @@ -1722,18 +1771,3 @@ SELECT max(Duration) AS DurationMax FROM traces GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; - -CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS -SELECT - OrgId, - Timestamp, - SessionId, - Seq, - Type AS Kind, - if(Type = 'navigation', '$pageview', Message) AS EventName, - domain(Url) AS Host, - path(Url) AS PagePath, - Url, - Attributes - FROM session_events - WHERE Type IN ('navigation', 'custom'); diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 200013018..78e001988 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7"; +pub const PROJECT_REVISION: &str = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "13"; +pub const SCHEMA_VERSION: &str = "16"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] @@ -78,9 +78,16 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "session_events", table: "session_events", - columns: &["OrgId", "SessionId", "Timestamp", "Seq", "Type", "Url", "TraceId", "Level", "Message", "TargetSelector", "TargetText", "NetMethod", "NetUrl", "NetStatus", "NetDurationMs", "ErrorStack", "Attributes"], - selects: &["__ORG__", "session_id", "timestamp", "seq", "type", "url", "trace_id", "level", "message", "target_selector", "target_text", "net_method", "net_url", "net_status", "net_duration_ms", "error_stack", "attributes"], - input_schema: "session_id String, timestamp DateTime64(9), seq UInt32, type LowCardinality(String), url String, trace_id String, level LowCardinality(String), message String, target_selector String, target_text String, net_method LowCardinality(String), net_url String, net_status UInt16, net_duration_ms UInt32, error_stack String, attributes Map(String, String)", + columns: &["OrgId", "SessionId", "Timestamp", "Seq", "Type", "Url", "TraceId", "Level", "Message", "TargetSelector", "TargetText", "NetMethod", "NetUrl", "NetStatus", "NetDurationMs", "ErrorStack", "Attributes", "VisitorId", "UserId", "GroupId"], + selects: &["__ORG__", "session_id", "timestamp", "seq", "type", "url", "trace_id", "level", "message", "target_selector", "target_text", "net_method", "net_url", "net_status", "net_duration_ms", "error_stack", "attributes", "visitor_id", "user_id", "group_id"], + input_schema: "session_id String, timestamp DateTime64(9), seq UInt32, type LowCardinality(String), url String, trace_id String, level LowCardinality(String), message String, target_selector String, target_text String, net_method LowCardinality(String), net_url String, net_status UInt16, net_duration_ms UInt32, error_stack String, attributes Map(String, String), visitor_id String, user_id String, group_id String", + }, + InsertMapping { + datasource: "product_events", + table: "product_events", + columns: &["OrgId", "Timestamp", "Source", "SessionId", "Seq", "VisitorId", "UserId", "GroupId", "Kind", "EventName", "Host", "PagePath", "Url", "ServiceName", "Attributes"], + selects: &["__ORG__", "timestamp", "source", "session_id", "seq", "visitor_id", "user_id", "group_id", "kind", "event_name", "host", "page_path", "url", "service_name", "attributes"], + input_schema: "timestamp DateTime64(9), source LowCardinality(String), session_id String, seq UInt32, visitor_id String, user_id String, group_id String, kind LowCardinality(String), event_name String, host LowCardinality(String), page_path String, url String, service_name LowCardinality(String), attributes Map(String, String)", }, ]; @@ -95,6 +102,7 @@ pub fn mapping_for(datasource: &str) -> Option<&'static InsertMapping> { "session_replays" => Some(&DATASOURCES[6]), "session_replay_events" => Some(&DATASOURCES[7]), "session_events" => Some(&DATASOURCES[8]), + "product_events" => Some(&DATASOURCES[9]), _ => None, } } diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md new file mode 100644 index 000000000..7c6f1aaad --- /dev/null +++ b/docs/product-events-funnels.md @@ -0,0 +1,210 @@ +# Product events + funnels — plan + +Status: **planned, not started** (2026-08-17). Goal: answer "referral → signed up → started a +plan" for any org (and for Maple itself), from browser, mobile and backend events, as a real +funnel in the product rather than a hand-written `run_sql`. + +## Where we are + +Verified against production on 2026-08-17: + +- `web_events` (`packages/domain/src/tinybird/datasources.ts`, `packages/query-engine/src/ch/tables.ts`) + already exists as "the funnel substrate": time-sorted `$pageview` + `track()` rows, `EventName` + as the step key, `Seq` tiebreak, `idx_event_name` skip index. **Nothing queries it as a funnel** — + no `windowFunnel`/`sequenceMatch` anywhere in the repo; the `funnel` widget is a name/value bar. +- Referral (`Referrer`, `ReferrerHost`, `Utm*`), `VisitorId`, `UserId`, `GroupId` live only on + `session_replays`. `web_events` carries `SessionId` only, so any cross-session question is a join. +- The cross-site visitor cookie works: `maple.dev` (`@maple-dev/browser`) and `app.maple.dev` + (effect-sdk client) share `VisitorId` and land in the same org. 4 weeks: 3,462 landing visitors → + 1,979 referred → 94 reached the app → 48 identified → 22 referred + identified. +- ~⅓ of `maple-web` sessions have empty `VisitorId`/`Host` (pre-0011 SDK builds, GPC); they drop + out of any visitor-keyed funnel. +- No `signup_completed` and no `plan_started` event exists. Plan start cannot come from the browser: + `attach` returns a Stripe `paymentUrl` and the tab leaves; the plan starts on the Stripe → Autumn + side. There is no server-side `track()` and no Clerk/Autumn/Stripe webhook route in `apps/api`. +- 30-day TTL on `session_replays` / `session_events` / `web_events`; `web_events` is MV-only and + cannot be rebuilt past its source's horizon. + +Today the referral → identified step is answerable with raw SQL; plan-start is not answerable at all. + +## Target model + +### 1. `product_events` (rename of `web_events`, widened) + +`web_events` is renamed because the table stops being web-only: browser rows still arrive via the +`session_events` MV, but backend and mobile events are **direct-ingested** into the same table. + +``` +product_events + OrgId LowCardinality(String) + Timestamp DateTime64(9) + Seq UInt32 -- browser: session_events.Seq; direct: 0 + Source LowCardinality(String) DEFAULT 'browser' -- browser | server | mobile + SessionId String DEFAULT '' -- '' for server events + VisitorId String DEFAULT '' -- device/anonymous id (browser cookie, mobile install id) + UserId String DEFAULT '' + GroupId String DEFAULT '' + Kind LowCardinality(String) -- navigation | custom | screen (mobile) + EventName String -- '$pageview' / '$screen' / track() name + Host LowCardinality(String) DEFAULT '' + PagePath String DEFAULT '' + Url String DEFAULT '' + ServiceName LowCardinality(String) DEFAULT '' -- which SDK/service emitted it + Attributes Map(String, String) +ENGINE MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) -- see note +TTL toDate(Timestamp) + INTERVAL 180 DAY +INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +``` + +Decisions baked in: + +- **Person key on the row.** The SDK already resolves identity lazily when session rows post and + spans start (`clerk-auth-bridge.tsx`, `identify()`); session events get the same stamping in + `packages/browser-session/src/events/events-sink.ts` (`visitor_id`, `user_id`, `group_id` on each + NDJSON line). The MV copies them through. No MV-side join against `session_replays` — that would + depend on insert ordering. + - Requires adding `VisitorId`/`UserId`/`GroupId` (`DEFAULT ''`) to `session_events` too + (migration + Tinybird forward query, same shape as the 0012 `Attributes` widening). + - Rows from older SDKs arrive with `''`; the funnel falls back to a `session_replays` lookup for + those, or simply excludes them — pick "exclude + show coverage" (matches how the analytics page + already treats sessions without the 0011 block). +- **Sorting key**: `Timestamp` second so time ranges are a primary-index scan (the reason + `web_events` exists), then `VisitorId` so a per-person `windowFunnel` groups over contiguous rows. + Keep `SessionId`/`Seq` for stable step order. +- **Retention 180d**, not 30. Table is tiny (13 distinct event names, thousands of rows/month in + the dogfood org). A referral→paid funnel spans weeks; 30d loses the tail. Browser rows can never + be rebuilt past 30d (source TTL) — accepted; direct-ingested rows have no source and are the + primary copy. `retention-matrix.test.ts` pins this; update it deliberately. +- **`Source` column** so a re-run of the browser backfill can `DELETE WHERE Source = 'browser'` + instead of `TRUNCATE` (which would destroy direct-ingested rows). This replaces the + "drop view → truncate → backfill → create view" one-writer invariant from migration 0014. +- Name: `product_events`. Not `events` (ambiguous next to `session_events`/`error_events`), not + `analytics_events` (the page is called Web Analytics; that name will age the same way). + +### 2. Direct ingest — `POST /v1/events` + +Ingest gateway (`apps/ingest/src/main.rs`), NDJSON like `/v1/sessionEvents`, authenticated by +ingest key (org from the key, never the body). Body per line: + +```json +{ "timestamp": "...", "name": "plan_started", "source": "server", + "user_id": "user_…", "group_id": "org_…", "visitor_id": "", "session_id": "", + "service_name": "maple-api", "attributes": { "plan": "startup" } } +``` + +- Same caps as `sanitize_session_event` (name 128, ≤32 props, key 64, value 1024, 8 KiB total). + Reject `name` starting with `$` from direct ingest (reserved for `$pageview`/`$screen`). +- `Kind = 'custom'` unless `name = '$screen'` (mobile screen views → `Kind='screen'`). +- New `TelemetrySignal::ProductEvents`, `INGEST_TINYBIRD_DATASOURCE_PRODUCT_EVENTS`, entry in + `clickhouse_insert_mappings.rs`, entitlement check reusing the browser-sessions feature id (or a + new `product_events` feature — billing decision, default: reuse). +- Writes go where session events go today (Tinybird managed; BYO CH via the export lane). +- Mobile: no SDK in scope. The endpoint *is* the contract; a mobile app posts with a persistent + install id as `visitor_id` and `identify`-equivalent `user_id`. `@maple-dev/effect-sdk` server side + gets `track()` (`packages/effect-sdk/src/server`?) as a thin client of this endpoint so Node/Bun + backends have the same call as the browser. + +### 3. Server-side emitters for Maple's own funnel + +`apps/api`: + +- `ProductEventsService.track({ userId, groupId, name, attributes })` — Effect service, posts to + the ingest gateway with Maple's own dogfood ingest key (same org that `maple-web`/`maple-landing` + report into). `Schema.TaggedError` for the failure; never fails the caller (fire-and-forget on a + forked fiber, `root: true` — see the ambient-span footgun). +- **`signup_completed`**: Clerk webhook `user.created` (Svix-signed) → new route + `apps/api/src/routes/webhooks/clerk.http.ts`. `user_id` from the payload; `visitor_id` unknown + server-side — stitching to the marketing visit happens via identity (below), not on this row. + Fallback/complement: `apps/web` fires `signup_completed` client-side on the first + `/quick-start` landing after Clerk `createdAt` is within N minutes; keep the webhook as truth. +- **`plan_started`**: webhook from the billing side (Autumn if it offers one for + attach/subscription-created; else Stripe `checkout.session.completed` + + `customer.subscription.created`, keyed back to the org via the Autumn customer id). Also emit + synchronously in the `attach` inline-success branch (no `paymentUrl`) for the no-redirect case; + dedup by `attributes.subscription_id`. +- Same route family later carries `plan_changed`, `plan_cancelled`. + +### 4. Identity stitching + +The person key for a funnel step is `if(UserId != '', UserId, VisitorId)`. Anonymous marketing +visits (VisitorId only) and server events (UserId only) meet through **`identity_links`**: an +MV over `session_replays` (and later mobile identify calls) emitting `(OrgId, VisitorId, UserId, +FirstSeen)` — ReplacingMergeTree keyed `(OrgId, VisitorId, UserId)`. The funnel query resolves +each row's person key via a `LEFT JOIN identity_links` (or `dictGet` on BYO CH) so a visitor's +pre-signup rows and their post-signup UserId rows collapse into one person. Cheap: the link table +is one row per (visitor,user) pair. + +### 5. Query engine + +`lib/clickhouse-builder`: +- Parametric aggregates `windowFunnel(windowSec, mode?)(ts, cond1..condN)` and + `sequenceMatch(pattern)(ts, cond…)` following the handwritten `quantile(q)` pattern in + `src/ch/functions/aggregate.ts:74`. `retention()` optional. + +`packages/query-engine/src/ch/queries/product-events.ts` (replaces `web-analytics.ts`'s +`web_events` references; the page-view queries move here unchanged): +- `productEventsFunnelQuery({ steps, keyBy: "person" | "visitor" | "user" | "session", + windowSeconds, filters })` → per-step `count`, `conversion_from_prev`, `conversion_from_first`. + Step = `{ eventName } | { pagePath, host? } | { referrerHost | utmSource | ... }`. A + session-dimension step (referral) becomes step 0's condition through the person's first + `session_replays` row; event steps are `EventName = …` conditions on `product_events`. +- `productEventsFunnelBreakdownQuery` — same, `GROUP BY` one dimension (`UtmSource`, + `ReferrerHost`, `Country`, `Attributes[k]`). +- `productEventNamesQuery` — the step picker's autocomplete (names + counts, 30d). +- Every query keeps `$.OrgId.eq(param.string("orgId"))`; register in `sql-catalog.ts` and add + parity coverage in the ClickHouse e2e like `web-analytics-parity.clickhouse.e2e.test.ts` + (seed dates must be now-relative — TTL). +- `apps/api/src/routes/internal/query-engine.http.ts` fallback message and the + `useWebEvents` switch get renamed (`useProductEvents`), same "absent table → raw + `session_events`" degrade for page views; funnels **require** the table (no raw fallback — + server/mobile rows only exist there). + +### 6. Surfaces + +- **Dashboard widget**: new `funnel` config that is step-based (not group-by). Today's `funnel` + render shape stays as the renderer; the *widget type* gains `steps[]`, `keyBy`, `window`. + `packages/domain/src/http/v2/dashboards.ts` + parity test, `dashboard-schema-doc.ts` for MCP. +- **`/analytics` → Funnels tab** reusing the existing filter sidebar (`WebAnalyticsFilters` become + `ProductEventsFilters`), plus an event-name breakdown panel. +- **MCP**: `query_funnel` tool (or a `funnel` mode on `query_data`) and `list_product_events`. +- Docs (`apps/landing/src/content/docs/…`): `track()` server-side, `/v1/events`, funnels page. + +## Migration / rename sequence + +Order matters because `web_events` has no dedup and `session_events` still holds every browser +row for 30d — the rename is a rebuild, not a `RENAME TABLE`. + +1. Migration `0016_product_events` (BYO CH) + Tinybird datasource `product_events` + MV + `product_events_mv` (from `session_events`, with the new columns), backfill spec from + `session_events` for the last 30d (same row-wise projection idea as 0014, `Source='browser'`). + Also `0016` adds `VisitorId/UserId/GroupId` to `session_events` (+ Tinybird forward query). +2. Local CLI: `local-schema-v6.sql`, `local-store-migrations/v5-to-v6-product-events.ts` + (+ test), bump the schema-version gate. +3. SDKs stamp identity on session events (`browser-session` events-sink; effect-sdk client + `track.ts`). Backwards compatible — defaults cover old builds. +4. Ingest: `/v1/events`, mappings, signal, env; deploy. +5. Query engine: flip page-view readers `web_events → product_events`; add funnel queries; + catalog + parity tests; `useWebEvents` rename. +6. Web/MCP surfaces. +7. `apps/api`: `ProductEventsService`, Clerk + billing webhooks, `attach` inline emit. +8. After one full deploy cycle with both tables populated and reads on the new one: migration + `0017_drop_web_events` (drop MV then table) and remove the Tinybird datasource; delete the + `web_events` local-schema entries going forward (old versions keep them for the migration chain). + +## Open decisions (defaults chosen; flag if you disagree) + +- Retention **180d** for `product_events` (could be 365; cost is negligible either way). +- Reuse the browser-sessions entitlement for `/v1/events` rather than a new billable feature. +- Person key = `UserId` else `VisitorId`, stitched through `identity_links`; no probabilistic + matching. +- `signup_completed` truth = Clerk webhook, not the client. +- Mobile SDK is **not** in scope; the HTTP contract is. + +## Quick win available before any of this + +Referral → identified in app → `onboarding_step_completed` as a `raw_sql_chart` widget over +`session_replays` + `web_events` (the join used for the numbers above). ~15 min, no schema change, +missing only the plan-start step. diff --git a/packages/domain/src/clickhouse/backfill.ts b/packages/domain/src/clickhouse/backfill.ts index 33f693734..fd671189f 100644 --- a/packages/domain/src/clickhouse/backfill.ts +++ b/packages/domain/src/clickhouse/backfill.ts @@ -52,6 +52,7 @@ export const SOURCE_TIME_COLUMNS: Readonly> = { service_overview_spans: "Timestamp", service_operations_minutely: "Minute", session_events: "Timestamp", + session_replays: "StartTime", } satisfies Readonly> const ident = (db: string, name: string): string => `\`${db}\`.\`${name}\`` diff --git a/packages/domain/src/clickhouse/migrations/0016_product_events.ts b/packages/domain/src/clickhouse/migrations/0016_product_events.ts new file mode 100644 index 000000000..2f3930de2 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0016_product_events.ts @@ -0,0 +1,168 @@ +import type { BackfillSpec } from "../backfill" + +/** + * The browser-row projection shared byte-for-byte between the live-write + * materialized view and {@link productEventsBrowserBackfill}. Two copies of this + * SELECT is two chances for a backfilled row and a live row of the same event to + * disagree, which on a table with no dedup surfaces as a page-view count that + * shifts at the backfill boundary. + */ +const PRODUCT_EVENTS_PROJECTION_SQL = `OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes` + +const PRODUCT_EVENTS_SOURCE_FILTER = "Type IN ('navigation', 'custom')" + +/** + * Browser-row backfill for {@link migration_0016_product_events}. + * + * Row-wise, so any chunk boundary is safe. Unlike 0014's `web_events`, the + * target is **dual-fed**: `POST /v1/events` writes server/mobile rows directly, + * with no source table to rebuild them from. So the idempotency step is not + * `TRUNCATE` but `DELETE WHERE Source = 'browser'` — the view is dropped first, + * only browser rows are cleared, the backfill runs, then the view is re-created. + * At no point are two writers of *browser* rows pointed at this table, and at no + * point can a directly ingested row be lost. + */ +export const productEventsBrowserBackfill: BackfillSpec = { + kind: "backfill", + target: "product_events", + columns: [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + ], + from: "session_events", + tsColumn: "Timestamp", + select: PRODUCT_EVENTS_PROJECTION_SQL, + where: PRODUCT_EVENTS_SOURCE_FILTER, +} + +/** + * Migration 0016 — product events: `web_events` becomes `product_events`. + * + * Three things, in order: + * + * 1. `session_events` gains `VisitorId`/`UserId`/`GroupId` (all `DEFAULT ''`), + * stamped per event by the SDK. Metadata-only `ADD COLUMN`s, like 0011. + * 2. `product_events` replaces `web_events`: same time-first fact table, plus + * `Source` (`browser`/`server`/`mobile`), the person key, `ServiceName`, and + * a 365-day TTL. Backends and mobile apps write it directly via + * `POST /v1/events`; browser rows arrive through `product_events_mv`. The + * browser half is backfilled from `session_events` (its 30-day window is all + * there is), and `web_events` + its view are dropped — every reader moved to + * the new table in the same release, and the old one cannot be rebuilt past + * what the new one now holds. + * 3. `identity_links` — (VisitorId, UserId) pairs out of `session_replays`, + * the stitch between a person's anonymous marketing visit and their + * identified/server-side events. Backfilled from `session_replays` for the + * same 30-day reason. + * + * Re-runnable by construction: views are dropped first, `product_events` + * clears only `Source = 'browser'` (never a directly ingested row), and + * `identity_links` is a ReplacingMergeTree so a re-insert of a pair is a no-op. + * + * **BYO ClickHouse only.** Managed orgs get `product_events_mv` / + * `identity_links_mv` via `tinybird deploy` from `materializations.ts`, and the + * populate is an explicit `tb` step at deploy time (see 0014's note; the SDK has + * no populate option). + * + * `requiredForIngest` stays at its default (true) this time: the ingest gateway + * writes `session_events` with the three new columns and `product_events` + * directly, and a cluster without them rejects those rows. + */ +export const migration_0016_product_events = { + version: 16, + description: + "Add identity columns to session_events; replace web_events with the dual-fed product_events table and add identity_links", + statements: [ + "ALTER TABLE session_events ADD COLUMN IF NOT EXISTS VisitorId String DEFAULT ''", + "ALTER TABLE session_events ADD COLUMN IF NOT EXISTS UserId String DEFAULT ''", + "ALTER TABLE session_events ADD COLUMN IF NOT EXISTS GroupId String DEFAULT ''", + "DROP VIEW IF EXISTS product_events_mv", + "DROP VIEW IF EXISTS identity_links_mv", + `CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY`, + `CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen DateTime64(9) +) +ENGINE = ReplacingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY`, + // Idempotency for the browser half only — a directly ingested row has no + // source to come back from. Lightweight delete; the view is already + // dropped, so nothing is writing browser rows while this runs. + "DELETE FROM product_events WHERE Source = 'browser'", + productEventsBrowserBackfill, + { + kind: "backfill", + target: "identity_links", + columns: ["OrgId", "VisitorId", "UserId", "FirstSeen"], + from: "session_replays", + tsColumn: "StartTime", + select: "OrgId, VisitorId, UserId, StartTime AS FirstSeen", + where: "VisitorId != '' AND UserId != ''", + } satisfies BackfillSpec, + // Views attached last; from here they are the only browser-row writers. + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT ${PRODUCT_EVENTS_PROJECTION_SQL} +FROM session_events +WHERE ${PRODUCT_EVENTS_SOURCE_FILTER}`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT OrgId, VisitorId, UserId, StartTime AS FirstSeen +FROM session_replays +WHERE VisitorId != '' AND UserId != ''`, + // The old table last, once the new one is populated and its writer live. + "DROP VIEW IF EXISTS web_events_mv", + "DROP TABLE IF EXISTS web_events", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index f0a215089..a3d77bce6 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -21,6 +21,7 @@ import { migration_0015_service_overview_minutely, serviceOverviewMinutelyBackfill, } from "./0015_service_overview_minutely" +import { migration_0016_product_events } from "./0016_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -35,14 +36,15 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) - expect(migrations.at(-1)).toBe(migration_0015_service_overview_minutely) - expect(latestMigrationVersion).toBe(15) - // 0010, 0014 and 0015 are performance-only, so the ingest-gating version - // skips all three and stays at 13 — nothing writes `web_events` or - // `service_overview_minutely` directly, and bumping it would un-ready every - // BYO-CH org's ingest routing for a read-path change. - expect(clickHouseSchemaVersion).toBe("13") + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) + expect(migrations.at(-1)).toBe(migration_0016_product_events) + expect(latestMigrationVersion).toBe(16) + // 0010, 0014 and 0015 are performance-only and skipped by the ingest-gating + // version; 0016 is not — the gateway writes `session_events`' new identity + // columns and `product_events` directly, so a BYO-CH org must apply it + // before ingest routes there again. + expect(clickHouseSchemaVersion).toBe("16") + expect(migration_0016_product_events.requiredForIngest).toBeUndefined() expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) expect(migration_0015_service_overview_minutely.requiredForIngest).toBe(false) diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 3c65b6337..9fbe6c4ae 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -14,6 +14,7 @@ import { migration_0012_session_event_attribute_keys } from "./0012_session_even import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" import { migration_0015_service_overview_minutely } from "./0015_service_overview_minutely" +import { migration_0016_product_events } from "./0016_product_events" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -60,6 +61,7 @@ export const migrations: ReadonlyArray = [ migration_0013_service_map_ingest_bridge, migration_0014_web_events, migration_0015_service_overview_minutely, + migration_0016_product_events, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/clickhouse/qualify.ts b/packages/domain/src/clickhouse/qualify.ts index 2b302520d..d162d9d91 100644 --- a/packages/domain/src/clickhouse/qualify.ts +++ b/packages/domain/src/clickhouse/qualify.ts @@ -15,6 +15,7 @@ export const CLICKHOUSE_MV_SOURCE_TABLES: ReadonlyArray = [ "service_overview_spans", "service_operations_minutely", "session_events", + "session_replays", ] /** diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index eab3088b7..e6caeeafc 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" as const +export const projectRevision = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -11,6 +11,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL Minute + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", + "CREATE TABLE IF NOT EXISTS identity_links (\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen DateTime64(9)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY tuple()\nORDER BY (OrgId, VisitorId, UserId)\nTTL toDate(FirstSeen) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS logs_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS metric_catalog (\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour)\nTTL Hour + INTERVAL 90 DAY", @@ -18,6 +19,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS metrics_gauge (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n BucketCounts Array(UInt64),\n ExplicitBounds Array(Float64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_sum (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n AggregationTemporality Int32,\n IsMonotonic Bool\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", + "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_external_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_map_children (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, ParentSpanId, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", @@ -33,7 +35,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS service_overview_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", - "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replays (\n OrgId LowCardinality(String),\n SessionId String,\n StartTime DateTime64(9),\n EndTime Nullable(DateTime64(9)),\n DurationMs Nullable(UInt32),\n Status LowCardinality(String),\n UserId String,\n UrlInitial String,\n UserAgent String,\n BrowserName LowCardinality(String),\n OsName LowCardinality(String),\n DeviceType LowCardinality(String),\n Country LowCardinality(String) DEFAULT '',\n ServiceName LowCardinality(String),\n PageViews UInt32 DEFAULT 0,\n ClickCount UInt32 DEFAULT 0,\n ErrorCount UInt32 DEFAULT 0,\n TraceIds Array(String) DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String),\n Version UInt32,\n VisitorId String DEFAULT '',\n VisitorIsNew UInt8 DEFAULT 0,\n UserEmail String DEFAULT '',\n UserName String DEFAULT '',\n GroupId String DEFAULT '',\n GroupName String DEFAULT '',\n UserTraits Map(String, String) DEFAULT map(),\n Referrer String DEFAULT '',\n ReferrerHost LowCardinality(String) DEFAULT '',\n UtmSource LowCardinality(String) DEFAULT '',\n UtmMedium LowCardinality(String) DEFAULT '',\n UtmCampaign LowCardinality(String) DEFAULT '',\n UtmTerm String DEFAULT '',\n UtmContent String DEFAULT '',\n Host LowCardinality(String) DEFAULT '',\n EntryPath String DEFAULT '',\n ExitPath String DEFAULT '',\n Language LowCardinality(String) DEFAULT '',\n LastActivityAt Nullable(DateTime64(9))\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(StartTime)\nORDER BY (OrgId, SessionId)\nTTL toDate(StartTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", @@ -41,11 +43,11 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", - "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'", + "CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS\nSELECT\n OrgId,\n VisitorId,\n UserId,\n StartTime AS FirstSeen\n FROM session_replays\n WHERE VisitorId != '' AND UserId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace", @@ -55,6 +57,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", @@ -81,5 +84,4 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv", - "CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS\nSELECT\n OrgId,\n Timestamp,\n SessionId,\n Seq,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", ] as const diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 42aaa991d..9de89811e 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" as const +export const projectRevision = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" as const export const datasources = [ { @@ -39,6 +39,11 @@ export const datasources = [ content: 'DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode=\'Error\' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', }, + { + name: "identity_links", + content: + 'DESCRIPTION >\n Visitor→user identity links, one row per (VisitorId, UserId) pair observed on a session_replays row with both set. Stitches anonymous and identified product_events into one person for funnels.\n\nSCHEMA >\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen DateTime64(9)\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "tuple()"\nENGINE_SORTING_KEY "OrgId, VisitorId, UserId"\nENGINE_TTL "toDate(FirstSeen) + INTERVAL 365 DAY"', + }, { name: "logs", content: @@ -74,6 +79,11 @@ export const datasources = [ content: 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`,\n IsMonotonic Bool `json:$.is_monotonic`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', }, + { + name: "product_events", + content: + "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map()\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4", + }, { name: "service_address_resolutions_hourly", content: @@ -152,7 +162,7 @@ export const datasources = [ { name: "session_events", content: - "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_type Type TYPE set(16) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, SessionId, Timestamp, Seq, Type, Url, TraceId, Level, Message,\n \t\tTargetSelector, TargetText, NetMethod, NetUrl, NetStatus, NetDurationMs,\n \t\tErrorStack,\n \t\tCAST(Attributes, 'Map(String, String)') AS Attributes", + "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes`,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_type Type TYPE set(16) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n \t\tOrgId, SessionId, Timestamp, Seq, Type, Url, TraceId, Level, Message,\n \t\tTargetSelector, TargetText, NetMethod, NetUrl, NetStatus, NetDurationMs,\n \t\tErrorStack,\n \t\tCAST(Attributes, 'Map(String, String)') AS Attributes", }, { name: "session_replay_events", @@ -189,11 +199,6 @@ export const datasources = [ content: 'DESCRIPTION >\n Hourly pre-aggregated trace metrics with sampling-weighted state columns. Generalized MV target for timeseries/breakdown/service-overview queries. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', }, - { - name: "web_events", - content: - 'DESCRIPTION >\n Web analytics fact table: navigation and custom events from session_events, sorted by time with domain(Url)/path(Url) pre-extracted. Powers page views, top pages and funnels. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, SessionId, Seq"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4', - }, ] as const export const pipes = [ @@ -217,6 +222,11 @@ export const pipes = [ content: "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans", }, + { + name: "identity_links_mv", + content: + "DESCRIPTION >\n Populates identity_links with every (VisitorId, UserId) pair observed on a session_replays row.\n\nNODE identity_links_mv_node\nSQL >\n SELECT\n OrgId,\n VisitorId,\n UserId,\n StartTime AS FirstSeen\n FROM session_replays\n WHERE VisitorId != '' AND UserId != ''\n\nTYPE MATERIALIZED\nDATASOURCE identity_links", + }, { name: "log_attribute_keys_mv", content: @@ -262,6 +272,11 @@ export const pipes = [ content: "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", }, + { + name: "product_events_mv", + content: + "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + }, { name: "service_external_edges_hourly_mv", content: @@ -392,11 +407,6 @@ export const pipes = [ content: "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly", }, - { - name: "web_events_mv", - content: - "DESCRIPTION >\n Populates web_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted and the event name normalized.\n\nNODE web_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n SessionId,\n Seq,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE web_events", - }, ] as const export const tinybirdProjectManifest = { diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 6e9c2ac21..fceea3167 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2026,6 +2026,14 @@ export const sessionEvents = defineDatasource("session_events", { Attributes: column(t.map(t.string(), t.string()), { jsonPath: "$.attributes", }), + // Identity, stamped by the SDK on every event (the same lazy `identify()` + // read that fills session rows and spans). All defaulted: an SDK build that + // predates them keeps writing, and `product_events_mv` copies them through + // so a funnel never needs an insert-order-dependent join back to + // `session_replays`. `''` means "unidentified", never "unknown". + VisitorId: column(t.string().default(""), { jsonPath: "$.visitor_id" }), + UserId: column(t.string().default(""), { jsonPath: "$.user_id" }), + GroupId: column(t.string().default(""), { jsonPath: "$.group_id" }), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", @@ -2062,76 +2070,97 @@ export const sessionEvents = defineDatasource("session_events", { export type SessionEventsRow = InferRow /** - * Web analytics fact table — the page views and custom events out of - * `session_events`, re-sorted by time and with the URL pre-parsed. - * Populated by materialized view, not direct ingestion. + * Product events fact table — every event a funnel or product-analytics query + * can step on, from every surface: browser page views and `track()` calls + * (materialized out of `session_events`), and events posted directly by + * backends and mobile apps via `POST /v1/events`. * - * ## Why this exists + * ## Why this exists (the browser half) * * `session_events` is sorted `(OrgId, SessionId, Timestamp, Seq)`, so a * time-range filter cannot use the primary index at all — only * `PARTITION BY toDate(Timestamp)` prunes, at day granularity. Its `idx_type` * skip index does not rescue that: navigation rows are interleaved with every - * session's clicks and network calls, so at `GRANULARITY 4` (32,768 rows per - * index granule) essentially every granule contains one and it prunes ~nothing. - * Measured on one production org over 7 days: 6,879 navigation rows out of - * 88,250 — the analytics queries read ~13x the data they use, and evaluated - * `domain(Url)`/`path(Url)` per row on top of it. - * - * A `host`/`pagePath` filter then multiplies that by twelve, because - * `navigationSessionsSubquery` is inlined into every branch of the breakdowns - * UNION. + * session's clicks and network calls, so at `GRANULARITY 4` essentially every + * granule contains one and it prunes ~nothing. Measured on one production org + * over 7 days: 6,879 navigation rows out of 88,250 — the analytics queries read + * ~13x the data they used, and evaluated `domain(Url)`/`path(Url)` per row. * * So: time-first sorting key, `Host`/`PagePath` materialized at write time, and - * only the two event types product analytics asks about. + * only the event types product analytics asks about (navigation + custom — + * clicks are ~2x more rows and a CSS selector is not a stable step definition). + * In-session debugging still reads `session_events` directly. + * + * ## Why it is dual-fed (the server/mobile half) * - * ## Why only navigation + custom + * "Started a plan" happens on a Stripe/webhook path the browser never sees, and + * a mobile app has no `session_events` transcript. Those events are posted + * straight into this table with `Source = 'server' | 'mobile'` and no + * `SessionId`. `Source` is also what keeps the browser backfill re-runnable: + * it deletes `WHERE Source = 'browser'` rather than truncating, so a re-run can + * never destroy directly ingested rows (which have no source to rebuild from). * - * These are the two that answer product questions and the two that can be funnel - * steps ("viewed /pricing", `track('signup_completed')`). Clicks are the next - * largest type by far (11,970 vs 6,879 over the same window) and a CSS selector - * is not a stable step definition, so including them would roughly triple the - * table to serve a worse funnel. In-session debugging still reads - * `session_events` directly — this table is not a replacement for it. + * ## Person key + * + * `VisitorId` (device/anonymous id — the browser's cross-subdomain cookie, a + * mobile install id) and `UserId`/`GroupId` from `identify()`, stamped on the + * row by the SDK. A funnel keys on `if(UserId != '', UserId, VisitorId)`, + * stitched across the anonymous→identified boundary by `identity_links`. + * `VisitorId` sits third in the sorting key so a per-person `windowFunnel` + * groups over contiguous rows inside the time range. * * ## Kind vs EventName * - * Both, deliberately. `track()` puts a caller-supplied name straight into - * `Message` with no reserved-prefix check, so a customer calling + * Both, deliberately. `track()` puts a caller-supplied name straight into the + * event with no reserved-prefix check on the browser path, so a customer calling * `track('$pageview')` would silently inflate page views if `EventName` were the * only discriminator. `Kind` is the column that is provably identical to the old * `Type = 'navigation'` predicate; `EventName` is the funnel's step key. * - * TTL 30 days, matching the rest of the session family. Note the source carries - * the same TTL, so this table can never be rebuilt past that horizon — a longer - * retention here is a one-way decision, not a tuning knob. + * TTL 365 days. The browser half can never be rebuilt past `session_events`' + * 30 days, but this table's rows are tiny (a handful of event names per org) and + * a referral → paid funnel spans weeks, so retention here is the primary copy, + * not a rebuildable cache. */ -export const webEvents = defineDatasource("web_events", { +export const productEvents = defineDatasource("product_events", { description: - "Web analytics fact table: navigation and custom events from session_events, sorted by time with domain(Url)/path(Url) pre-extracted. Powers page views, top pages and funnels. Populated by materialized view.", - jsonPaths: false, + "Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.", schema: { - OrgId: t.string().lowCardinality(), - Timestamp: t.dateTime64(9), - SessionId: t.string(), - /** Breaks ties within a millisecond, so a funnel's step order is stable. */ - Seq: t.uint32(), - /** `navigation` | `custom` — the source `Type`, carried through unchanged. */ - Kind: t.string().lowCardinality(), - /** `$pageview` for navigation, else the `track()` name. */ - EventName: t.string(), + OrgId: column(t.string().lowCardinality(), { jsonPath: "$.org_id" }), + Timestamp: column(t.dateTime64(9), { jsonPath: "$.timestamp" }), + /** `browser` (from session_events) | `server` | `mobile`. */ + Source: column(t.string().lowCardinality().default("browser"), { + jsonPath: "$.source", + }), + /** Empty for server events. */ + SessionId: column(t.string().default(""), { jsonPath: "$.session_id" }), + /** Breaks ties within a millisecond, so a funnel's step order is stable. 0 for direct rows. */ + Seq: column(t.uint32().default(0), { jsonPath: "$.seq" }), + VisitorId: column(t.string().default(""), { jsonPath: "$.visitor_id" }), + UserId: column(t.string().default(""), { jsonPath: "$.user_id" }), + GroupId: column(t.string().default(""), { jsonPath: "$.group_id" }), + /** `navigation` | `custom` | `screen` — the source type, carried through unchanged. */ + Kind: column(t.string().lowCardinality(), { jsonPath: "$.kind" }), + /** `$pageview` for navigation, `$screen` for mobile screen views, else the `track()` name. */ + EventName: column(t.string(), { jsonPath: "$.event_name" }), /** `domain(Url)`. LowCardinality: an org has a handful of hosts. */ - Host: t.string().lowCardinality(), + Host: column(t.string().lowCardinality().default(""), { jsonPath: "$.host" }), /** `path(Url)` — pathname only, so no query string or fragment. Unbounded for `/orders/:uuid` apps, hence plain String. */ - PagePath: t.string(), - Url: t.string(), + PagePath: column(t.string().default(""), { jsonPath: "$.page_path" }), + Url: column(t.string().default(""), { jsonPath: "$.url" }), + /** The emitting service (`maple-api`, `acme-ios`). Empty on browser rows — the session carries it. */ + ServiceName: column(t.string().lowCardinality().default(""), { + jsonPath: "$.service_name", + }), /** `track()` props. Plain String keys — the customer's app chooses them. */ - Attributes: t.map(t.string(), t.string()), + Attributes: column(t.map(t.string(), t.string()).defaultExpr("map()"), { + jsonPath: "$.attributes", + }), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", - sortingKey: ["OrgId", "Timestamp", "SessionId", "Seq"], - ttl: "toDate(Timestamp) + INTERVAL 30 DAY", + sortingKey: ["OrgId", "Timestamp", "VisitorId", "SessionId", "Seq"], + ttl: "toDate(Timestamp) + INTERVAL 365 DAY", }), indexes: [ { @@ -2144,7 +2173,47 @@ export const webEvents = defineDatasource("web_events", { type: "set(64)", granularity: 4, }, + { + // "Everything this user did" — the person drill-in and the + // UserId-keyed funnel branch. Near-unique values, so a bloom filter. + name: "idx_user_id", + expr: "UserId", + type: "bloom_filter", + granularity: 4, + }, ], }) -export type WebEventsRow = InferRow +export type ProductEventsRow = InferRow + +/** + * Identity links — one row per (visitor, user) pair ever observed together on a + * session, materialized out of `session_replays` (later also from mobile + * `identify` calls). This is how a funnel collapses a person's anonymous + * marketing visit (VisitorId only) and their later server-side events (UserId + * only) into one row: a `product_events` row resolves its person as + * `if(UserId != '', UserId, coalesce(link.UserId, VisitorId))`. + * + * ReplacingMergeTree keyed on the pair, so re-observing it is a no-op; the + * lowest `FirstSeen` wins on merge because the reader takes `min()`. + */ +export const identityLinks = defineDatasource("identity_links", { + description: + "Visitor→user identity links, one row per (VisitorId, UserId) pair observed on a session_replays row with both set. Stitches anonymous and identified product_events into one person for funnels.", + jsonPaths: false, + schema: { + OrgId: t.string().lowCardinality(), + VisitorId: t.string(), + UserId: t.string(), + FirstSeen: t.dateTime64(9), + }, + engine: engine.replacingMergeTree({ + partitionKey: "tuple()", + sortingKey: ["OrgId", "VisitorId", "UserId"], + // A pair not re-observed for a year is dead weight; the reader takes the + // most recent link anyway. + ttl: "toDate(FirstSeen) + INTERVAL 365 DAY", + }), +}) + +export type IdentityLinksRow = InferRow diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index ea9bc3a52..1d24c5a39 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -25,7 +25,9 @@ import { spanMetricsCallsHourly, serviceOperationsMinutely, serviceOperationsHourly, - webEvents, + productEvents, + identityLinks, + sessionReplays, } from "./datasources" import { DB_NAMESPACE_ATTR_SQL, @@ -1421,7 +1423,7 @@ export const logsAggregatesHourlyMv = defineMaterializedView("logs_aggregates_ho }) /** - * Populates `web_events` — the web analytics fact table. + * Populates the browser half of `product_events` — the product events fact table. * * A pure row-wise projection: filter to the two product-analytics event types, * pre-extract `domain(Url)`/`path(Url)`, re-sort by time in the target. No @@ -1442,27 +1444,37 @@ export const logsAggregatesHourlyMv = defineMaterializedView("logs_aggregates_ho * the page-view predicate stays provably identical to the pre-rollup * `Type = 'navigation'` even if a customer calls `track('$pageview')`. * - * Column order must match the `web_events` SCHEMA order — enforced by + * `Source` is the literal `'browser'`: this view is the only writer of browser + * rows, and the backfill deletes by it. Identity columns are copied through from + * the SDK-stamped `session_events` row — never joined from `session_replays`, + * whose v1/v2 rows may land after the event. + * + * Column order must match the `product_events` SCHEMA order — enforced by * `materialized-projection-order.test.ts`. */ -export const webEventsMv = defineMaterializedView("web_events_mv", { +export const productEventsMv = defineMaterializedView("product_events_mv", { description: - "Populates web_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted and the event name normalized.", - datasource: webEvents, + "Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.", + datasource: productEvents, nodes: [ node({ - name: "web_events_mv_node", + name: "product_events_mv_node", sql: ` SELECT OrgId, Timestamp, + 'browser' AS Source, SessionId, Seq, + VisitorId, + UserId, + GroupId, Type AS Kind, if(Type = 'navigation', '$pageview', Message) AS EventName, domain(Url) AS Host, path(Url) AS PagePath, Url, + '' AS ServiceName, Attributes FROM session_events WHERE Type IN ('navigation', 'custom') @@ -1470,3 +1482,32 @@ export const webEventsMv = defineMaterializedView("web_events_mv", { }), ], }) + +/** + * Populates `identity_links` from `session_replays` rows that carry both a + * visitor and a user id. + * + * Per-block firing is harmless here for the same reason it is fatal for + * per-session aggregates: this is a pure filter+project of one row, and the + * target is a ReplacingMergeTree keyed on the pair, so seeing the v1 and v2 rows + * of one session just re-inserts the same link. + */ +export const identityLinksMv = defineMaterializedView("identity_links_mv", { + description: + "Populates identity_links with every (VisitorId, UserId) pair observed on a session_replays row.", + datasource: identityLinks, + nodes: [ + node({ + name: "identity_links_mv_node", + sql: ` + SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != '' + `, + }), + ], +}) diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index 4bec7be5e..198fae613 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -41,9 +41,10 @@ const RETENTION_DAYS = { trace_list_mv: 30, traces: 30, traces_aggregates_hourly: 365, - // Pinned to its source `session_events`: past 30 days this table could never - // be rebuilt, so a longer tier here is a retention decision, not a tuning knob. - web_events: 30, + // Browser rows past 30 days can never be rebuilt from `session_events`, but + // direct-ingested (server/mobile) rows have no source: this table IS the copy. + product_events: 365, + identity_links: 365, } as const const ZERO_RETENTION_DATASOURCES = ["service_map_edges_hourly_ingest"] as const diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 4d8bdfc82..e798b6294 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -652,10 +652,16 @@ export const SessionEvents = table("session_events", { ErrorStack: T.string, // Overflow / extensibility. Attributes: T.map(T.string, T.string), + // Identity stamped by the SDK per event ('' on builds that predate it). + VisitorId: T.string, + UserId: T.string, + GroupId: T.string, }) -// Web analytics fact table — the navigation and custom rows of session_events, -// re-sorted by time with the URL pre-parsed. Populated by `web_events_mv`. +// Product events fact table — every event a funnel or product-analytics query +// can step on. Dual-fed: the navigation and custom rows of session_events arrive +// via `product_events_mv` (Source='browser'), backend and mobile events are +// posted directly through `POST /v1/events` (Source='server' | 'mobile'). // // Exists because session_events is sorted (OrgId, SessionId, Timestamp, Seq), so // a time-range filter there cannot use the primary index at all, and its idx_type @@ -663,29 +669,51 @@ export const SessionEvents = table("session_events", { // session's transcript). Reads that only want page views were scanning ~13x the // rows they used and parsing domain(Url)/path(Url) per row on top. // -// Also the funnel substrate: windowFunnel over (Timestamp, EventName, PagePath) -// grouped by SessionId. -export const WebEvents = table("web_events", { +// The funnel substrate: windowFunnel over (Timestamp, EventName, PagePath) +// grouped by the person key `if(UserId != '', UserId, VisitorId)`, stitched +// through `identity_links`. VisitorId is third in the sorting key so one +// person's rows are contiguous inside a time range. +export const ProductEvents = table("product_events", { OrgId: T.string, Timestamp: T.dateTime64, + // "browser" | "server" | "mobile". + Source: T.string, + // '' on server rows. SessionId: T.string, - // Tiebreaker within a millisecond, so a funnel's step order is stable. + // Tiebreaker within a millisecond, so a funnel's step order is stable. 0 on + // direct rows. Seq: T.uint32, - // "navigation" | "custom" — the source Type, carried through unchanged. This - // is the page-view predicate, NOT `EventName = '$pageview'`: track() takes a - // caller-supplied name with no reserved-prefix check, so a customer calling - // track('$pageview') would otherwise inflate the count. + VisitorId: T.string, + UserId: T.string, + GroupId: T.string, + // "navigation" | "custom" | "screen" — the source type, carried through + // unchanged. This is the page-view predicate, NOT `EventName = '$pageview'`: + // track() takes a caller-supplied name with no reserved-prefix check on the + // browser path, so a customer calling track('$pageview') would otherwise + // inflate the count. Kind: T.string, - // "$pageview" for navigation, else the track() name. The funnel step key. + // "$pageview" for navigation, "$screen" for mobile screens, else the track() + // name. The funnel step key. EventName: T.string, // domain(Url) / path(Url), materialized at write time. Host: T.string, PagePath: T.string, Url: T.string, + // The emitting service on direct rows; '' on browser rows. + ServiceName: T.string, // track() props. Attributes: T.map(T.string, T.string), }) +// (VisitorId, UserId) pairs observed together on a session_replays row. +// ReplacingMergeTree keyed on the pair — always aggregate (`min(FirstSeen)`) or +// semi-join; never assume one row per pair on read. +export const IdentityLinks = table("identity_links", { + OrgId: T.string, + VisitorId: T.string, + UserId: T.string, + FirstSeen: T.dateTime64, +}) export const MetricsExpHistogram = table("metrics_exponential_histogram", { OrgId: T.string, ResourceAttributes: T.map(T.string, T.string), diff --git a/scripts/generate-clickhouse-insert-mappings.ts b/scripts/generate-clickhouse-insert-mappings.ts index c639668bb..55cb49fb3 100644 --- a/scripts/generate-clickhouse-insert-mappings.ts +++ b/scripts/generate-clickhouse-insert-mappings.ts @@ -24,6 +24,7 @@ const INGEST_DATASOURCES = [ "session_replays", "session_replay_events", "session_events", + "product_events", ] as const // Replaced by the Rust binary with the pinned, escaped org-id string literal. From 58b4b9c262c48f041f06f78a881fe24b7497a741 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:12:00 +0200 Subject: [PATCH 02/15] feat(ingest): POST /v1/events writes product_events directly NDJSON, ingest-key auth, org from the key. Rows are rebuilt from a sanitized allowlist (name/timestamp/source/identity/url/attributes); malformed rows are dropped individually. New TelemetrySignal::ProductEvents rides the existing native-rows pipeline, so BYO-ClickHouse export works via the generated product_events mapping. --- apps/ingest/benches/ingest_bench.rs | 1 + apps/ingest/src/main.rs | 155 +++++- apps/ingest/src/session_analytics.rs | 441 ++++++++++++++++++ apps/ingest/src/telemetry.rs | 12 + .../docs/session-replay/product-events-api.md | 68 +++ 5 files changed, 676 insertions(+), 1 deletion(-) create mode 100644 apps/landing/src/content/docs/session-replay/product-events-api.md diff --git a/apps/ingest/benches/ingest_bench.rs b/apps/ingest/benches/ingest_bench.rs index b87348101..88bc7e3fb 100644 --- a/apps/ingest/benches/ingest_bench.rs +++ b/apps/ingest/benches/ingest_bench.rs @@ -121,6 +121,7 @@ impl BenchFixture { datasource_session_replays: "session_replays".to_string(), datasource_session_replay_events: "session_replay_events".to_string(), datasource_session_events: "session_events".to_string(), + datasource_product_events: "product_events".to_string(), }, Client::builder() .timeout(Duration::from_secs(5)) diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 8e84b3790..0a41ed5e8 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -40,7 +40,7 @@ use maple_ingest::otel::{ use maple_ingest::otlp_json; use maple_ingest::r2::{replay_object_key, ReplayBlobStore}; use maple_ingest::session_analytics::{ - derive_referrer_host, sanitize_session_event, sanitize_session_meta, + derive_referrer_host, sanitize_product_event, sanitize_session_event, sanitize_session_meta, }; use maple_ingest::telemetry::{ AttributeMappingRule, ClickHouseBreakerConfig, ClickHouseTarget, ClickHouseTargetProvider, @@ -315,6 +315,8 @@ impl AppConfig { .unwrap_or_else(|_| "session_replay_events".to_string()), datasource_session_events: std::env::var("INGEST_TINYBIRD_DATASOURCE_SESSION_EVENTS") .unwrap_or_else(|_| "session_events".to_string()), + datasource_product_events: std::env::var("INGEST_TINYBIRD_DATASOURCE_PRODUCT_EVENTS") + .unwrap_or_else(|_| "product_events".to_string()), }; if write_mode.uses_tinybird() { tinybird.validate()?; @@ -1679,6 +1681,7 @@ async fn main() { .route("/v1/sessionReplays/meta", post(handle_replay_meta)) .route("/v1/sessionReplays/blob", post(handle_replay_blob)) .route("/v1/sessionEvents", post(handle_session_events)) + .route("/v1/events", post(handle_product_events)) .route( "/v1/logpush/cloudflare/http_requests/{connector_id}", post(handle_cloudflare_logpush_http_requests), @@ -2578,6 +2581,155 @@ async fn handle_session_events_inner( Ok(count) } +async fn handle_product_events( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + metrics::request_started(); + let _guard = InFlightGuard; + let span = tracing::info_span!( + "ingest_product_events", + otel.name = "POST /v1/events", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + otel.status_description = tracing::field::Empty, + "maple.ingest.reject_reason" = tracing::field::Empty, + "http.request.method" = "POST", + "http.route" = "/v1/events", + "http.request.body.size" = body.len(), + "http.response.status_code" = tracing::field::Empty, + "error.type" = tracing::field::Empty, + "maple.signal" = "product_events", + "maple.org_id" = tracing::field::Empty, + "maple.ingest.clickhouse_ready" = tracing::field::Empty, + "maple.ingest.destination" = tracing::field::Empty, + "maple.product_events.dropped" = tracing::field::Empty, + ); + let span_handle = span.clone(); + match handle_product_events_inner(&state, &headers, body) + .instrument(span) + .await + { + Ok(count) => { + span_handle.record("http.response.status_code", 200u16); + span_handle.record("otel.status_code", "Ok"); + (StatusCode::OK, axum::Json(AcceptedBody { accepted: count })).into_response() + } + Err(error) => { + let status = error.status.as_u16(); + span_handle.record("http.response.status_code", status); + span_handle.record("error.type", error.error_kind()); + record_rejection_reason( + &span_handle, + status, + error.error_kind(), + error.message.as_str(), + ); + error.into_response() + } + } +} + +/// `POST /v1/events` — product events posted directly by backends and mobile +/// apps (browser rows reach `product_events` through the `session_events` +/// materialized view instead). Same auth, entitlement gate, NDJSON framing and +/// per-row drop policy as `/v1/sessionEvents`; the row shape is fixed by +/// `sanitize_product_event`. +async fn handle_product_events_inner( + state: &AppState, + headers: &HeaderMap, + body: Bytes, +) -> Result { + let resolved_key = match resolve_replay_key(state, headers).await? { + Some(resolved_key) => resolved_key, + None => return Ok(0), + }; + let org_id = resolved_key.org_id.clone(); + Span::current().record("maple.org_id", org_id.as_str()); + Span::current().record( + "maple.ingest.clickhouse_ready", + resolved_key.clickhouse_ready, + ); + let destination = native_destination_for(&resolved_key); + Span::current().record("maple.ingest.destination", destination.as_str()); + + // Product events reuse the browser-sessions entitlement (the plan doc's + // default) and, like session events, are not separately metered — a + // `product_events` meter is a pricing decision, not a schema one. + if org_id != SENTINEL_ORG_ID { + if let Some(error) = + entitlement_rejection(state, &org_id, BROWSER_SESSIONS_FEATURE_ID).await + { + return Err(error); + } + } + + let pipeline = native_rows_pipeline_for( + state, + destination, + "Product event storage is not configured", + )?; + + // One receipt time for the whole batch: rows without a `timestamp` all + // land at the moment the request arrived, not spread across the parse. + let received_at = chrono::Utc::now(); + let mut rows: Vec> = Vec::new(); + let mut dropped: u64 = 0; + for line in body.split(|&b| b == b'\n') { + if line.iter().all(u8::is_ascii_whitespace) { + continue; + } + let mut value: serde_json::Value = serde_json::from_slice(line) + .map_err(|e| ApiError::bad_request(format!("invalid product event JSON: {e}")))?; + let obj = value + .as_object_mut() + .ok_or_else(|| ApiError::bad_request("product event must be a JSON object"))?; + if !sanitize_product_event(obj, received_at) { + dropped += 1; + continue; + } + // org_id comes from the authenticated key, never the body — the + // sanitizer already discarded whatever the client sent under that name. + obj.insert( + "org_id".to_string(), + serde_json::Value::String(org_id.clone()), + ); + rows.push( + serde_json::to_vec(&value) + .map_err(|e| ApiError::bad_request(format!("failed to re-serialize event: {e}")))?, + ); + } + + if dropped > 0 { + Span::current().record("maple.product_events.dropped", dropped); + warn!( + org_id = %org_id, + dropped, + "dropped malformed product events (name, source or timestamp)" + ); + } + + if rows.is_empty() { + return Ok(0); + } + let count = rows.len(); + pipeline + .accept_rows_to( + &org_id, + state.config.tinybird.datasource_product_events.clone(), + rows, + TelemetrySignal::ProductEvents, + destination, + ) + .await + .map_err(|e| { + warn!(org_id = %org_id, error = %e, "product events enqueue rejected"); + api_error_from_pipeline(&e) + })?; + Ok(count) +} + /// Decompressed length of a gzip payload, without materializing it. /// /// Same number `read_to_string(...).len()` would produce, and the same @@ -6152,6 +6304,7 @@ mod tests { datasource_session_replays: "session_replays".to_string(), datasource_session_replay_events: "session_replay_events".to_string(), datasource_session_events: "session_events".to_string(), + datasource_product_events: "product_events".to_string(), } } diff --git a/apps/ingest/src/session_analytics.rs b/apps/ingest/src/session_analytics.rs index dfd9e56a0..dd1217fa8 100644 --- a/apps/ingest/src/session_analytics.rs +++ b/apps/ingest/src/session_analytics.rs @@ -241,6 +241,216 @@ pub fn sanitize_session_event(obj: &mut serde_json::Map Option> { + let trimmed = value.trim(); + if let Ok(parsed) = chrono::DateTime::parse_from_rfc3339(trimmed) { + return Some(parsed.with_timezone(&chrono::Utc)); + } + chrono::NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%d %H:%M:%S%.f") + .ok() + .map(|naive| naive.and_utc()) +} + +fn format_product_event_timestamp(at: chrono::DateTime) -> String { + at.format(PRODUCT_EVENT_TIMESTAMP_FORMAT).to_string() +} + +/// `(host, page_path)` for a product-event `url`, matching what +/// `product_events_mv` computes for browser rows with ClickHouse's +/// `domain(Url)` / `path(Url)`: the lowercase host without port or userinfo, +/// and the pathname without query string or fragment. A URL that does not +/// parse as absolute contributes no host, and its path is whatever precedes +/// the first `?`/`#` — the same shape `path()` yields for a bare `/pricing`. +fn derive_url_parts(url: &str) -> (String, String) { + if let Ok(parsed) = url::Url::parse(url) { + if let Some(host) = parsed.host_str() { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + return (host, parsed.path().to_string()); + } + } + let end = url.find(['?', '#']).unwrap_or(url.len()); + (String::new(), url[..end].to_string()) +} + +/// Optional string field from an untrusted object: `None` when absent, blank, +/// or not a string. Off-type values are treated as absent rather than +/// stringified so a JSON number never becomes an identifier by accident. +fn optional_str<'a>( + obj: &'a serde_json::Map, + field: &str, +) -> Option<&'a str> { + obj.get(field) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) +} + +/// Rewrite one direct-ingested product event (`POST /v1/events`) into the exact +/// row shape `product_events` is written in. Returns false when the row must be +/// dropped: missing/over-long/reserved `name`, unknown `source`, or an +/// unparseable `timestamp`. +/// +/// On success `obj` holds only the output fields (`timestamp`, `source`, +/// `session_id`, `seq`, `visitor_id`, `user_id`, `group_id`, `kind`, +/// `event_name`, `host`, `page_path`, `url`, `service_name`, `attributes`) — +/// the caller adds `org_id` from the authenticated key. As with session events, +/// a bad row is counted and skipped rather than failing the batch: a backend +/// flushing a buffer of events must not lose the good ones to one malformed +/// line. +pub fn sanitize_product_event( + obj: &mut serde_json::Map, + received_at: chrono::DateTime, +) -> bool { + let Some(name) = optional_str(obj, "name") else { + return false; + }; + if name.len() > PRODUCT_EVENT_MAX_NAME_BYTES { + // Truncating a step key would silently mint a *different* event name; + // dropping keeps `EventName` exactly what the caller intended, or nothing. + return false; + } + if name.starts_with('$') && name != PRODUCT_EVENT_SCREEN_NAME { + return false; + } + let kind = if name == PRODUCT_EVENT_SCREEN_NAME { + "screen" + } else { + "custom" + }; + let name = name.to_string(); + + let source = match obj.get("source") { + None | Some(serde_json::Value::Null) => "server".to_string(), + Some(serde_json::Value::String(source)) + if PRODUCT_EVENT_SOURCES.contains(&source.as_str()) => + { + source.clone() + } + Some(_) => return false, + }; + + let timestamp = match obj.get("timestamp") { + None | Some(serde_json::Value::Null) => received_at, + Some(serde_json::Value::String(raw)) => match parse_product_event_timestamp(raw) { + Some(parsed) => parsed, + None => return false, + }, + Some(_) => return false, + }; + + let url = optional_str(obj, "url") + .map(|u| truncate_str(u, PRODUCT_EVENT_MAX_URL_BYTES)) + .unwrap_or_default(); + let (derived_host, derived_path) = if url.is_empty() { + (String::new(), String::new()) + } else { + derive_url_parts(&url) + }; + // An explicit `page_path` wins over the one derived from `url`: mobile + // `$screen` events have no URL and carry the screen name here. + let page_path = optional_str(obj, "page_path") + .map(str::to_string) + .unwrap_or(derived_path); + + let mut row = serde_json::Map::new(); + row.insert( + "timestamp".to_string(), + serde_json::Value::String(format_product_event_timestamp(timestamp)), + ); + row.insert("source".to_string(), serde_json::Value::String(source)); + for (field, max_bytes) in PRODUCT_EVENT_ID_FIELDS { + let value = optional_str(obj, field) + .map(|v| truncate_str(v, max_bytes)) + .unwrap_or_default(); + row.insert(field.to_string(), serde_json::Value::String(value)); + } + row.insert("seq".to_string(), serde_json::json!(0)); + row.insert( + "kind".to_string(), + serde_json::Value::String(kind.to_string()), + ); + row.insert("event_name".to_string(), serde_json::Value::String(name)); + row.insert( + "host".to_string(), + serde_json::Value::String(truncate_str(&derived_host, PRODUCT_EVENT_MAX_HOST_BYTES)), + ); + row.insert( + "page_path".to_string(), + serde_json::Value::String(truncate_str(&page_path, PRODUCT_EVENT_MAX_PAGE_PATH_BYTES)), + ); + row.insert("url".to_string(), serde_json::Value::String(url)); + + let attributes = match obj.remove("attributes") { + Some(serde_json::Value::Object(mut attributes)) => { + // Same caps as `track()` props on session events: customer-chosen + // keys into a Map(String, String) column. + clamp_string_map( + &mut attributes, + SESSION_EVENT_MAX_ATTRIBUTES, + SESSION_EVENT_MAX_ATTRIBUTE_KEY_BYTES, + SESSION_EVENT_MAX_ATTRIBUTE_VALUE_BYTES, + ); + attributes + } + // Off-type `attributes` fall back to the column default rather than + // quarantining the row. + _ => serde_json::Map::new(), + }; + row.insert( + "attributes".to_string(), + serde_json::Value::Object(attributes), + ); + + *obj = row; + true +} + #[cfg(test)] mod tests { use super::*; @@ -419,4 +629,235 @@ mod tests { let truncated = truncate_str(&value, 5); assert_eq!(truncated, "éé"); } + + fn received_at() -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339("2026-08-17T12:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc) + } + + fn product_event(json: serde_json::Value) -> serde_json::Map { + json.as_object().cloned().unwrap() + } + + #[test] + fn product_event_name_rules() { + // Plain custom name → kind custom, event_name verbatim. + let mut ok = product_event(serde_json::json!({ "name": "plan_started" })); + assert!(sanitize_product_event(&mut ok, received_at())); + assert_eq!(ok["event_name"], "plan_started"); + assert_eq!(ok["kind"], "custom"); + + // `$screen` is the one reserved name direct ingest accepts. + let mut screen = product_event( + serde_json::json!({ "name": "$screen", "source": "mobile", "page_path": "Checkout" }), + ); + assert!(sanitize_product_event(&mut screen, received_at())); + assert_eq!(screen["kind"], "screen"); + assert_eq!(screen["event_name"], "$screen"); + assert_eq!(screen["page_path"], "Checkout"); + + for rejected in [ + serde_json::json!({}), + serde_json::json!({ "name": "" }), + serde_json::json!({ "name": " " }), + serde_json::json!({ "name": 42 }), + serde_json::json!({ "name": "$pageview" }), + serde_json::json!({ "name": "$identify" }), + serde_json::json!({ "name": "x".repeat(PRODUCT_EVENT_MAX_NAME_BYTES + 1) }), + ] { + let mut obj = product_event(rejected.clone()); + assert!( + !sanitize_product_event(&mut obj, received_at()), + "{rejected} should be dropped" + ); + } + + let mut max = product_event( + serde_json::json!({ "name": "x".repeat(PRODUCT_EVENT_MAX_NAME_BYTES) }), + ); + assert!(sanitize_product_event(&mut max, received_at())); + } + + #[test] + fn product_event_source_defaults_to_server_and_rejects_unknown() { + let mut default = product_event(serde_json::json!({ "name": "signup_completed" })); + assert!(sanitize_product_event(&mut default, received_at())); + assert_eq!(default["source"], "server"); + + let mut mobile = + product_event(serde_json::json!({ "name": "purchase", "source": "mobile" })); + assert!(sanitize_product_event(&mut mobile, received_at())); + assert_eq!(mobile["source"], "mobile"); + + for source in [ + serde_json::json!("browser"), + serde_json::json!("Server"), + serde_json::json!(""), + serde_json::json!(1), + ] { + let mut obj = product_event(serde_json::json!({ "name": "purchase", "source": source })); + assert!( + !sanitize_product_event(&mut obj, received_at()), + "source {source} should be dropped" + ); + } + } + + #[test] + fn product_event_timestamp_is_normalized_to_clickhouse_form() { + // Missing → gateway receipt time. + let mut missing = product_event(serde_json::json!({ "name": "e" })); + assert!(sanitize_product_event(&mut missing, received_at())); + assert_eq!(missing["timestamp"], "2026-08-17 12:00:00.000000000"); + + // RFC 3339 with an offset is converted to UTC. + let mut rfc = product_event( + serde_json::json!({ "name": "e", "timestamp": "2026-08-17T14:15:30.123+02:00" }), + ); + assert!(sanitize_product_event(&mut rfc, received_at())); + assert_eq!(rfc["timestamp"], "2026-08-17 12:15:30.123000000"); + + // ClickHouse text form, with and without fractional seconds. + let mut ch = product_event( + serde_json::json!({ "name": "e", "timestamp": "2026-08-17 10:15:30.123" }), + ); + assert!(sanitize_product_event(&mut ch, received_at())); + assert_eq!(ch["timestamp"], "2026-08-17 10:15:30.123000000"); + let mut ch_whole = + product_event(serde_json::json!({ "name": "e", "timestamp": "2026-08-17 10:15:30" })); + assert!(sanitize_product_event(&mut ch_whole, received_at())); + assert_eq!(ch_whole["timestamp"], "2026-08-17 10:15:30.000000000"); + + for bad in [ + serde_json::json!("yesterday"), + serde_json::json!("1755432000000"), + serde_json::json!(1755432000000u64), + ] { + let mut obj = product_event(serde_json::json!({ "name": "e", "timestamp": bad })); + assert!( + !sanitize_product_event(&mut obj, received_at()), + "timestamp {bad} should be dropped" + ); + } + } + + #[test] + fn product_event_url_derives_host_and_page_path() { + let mut obj = product_event(serde_json::json!({ + "name": "checkout_viewed", + "url": "https://App.Example.COM:8443/checkout/123?step=2#pay", + })); + assert!(sanitize_product_event(&mut obj, received_at())); + assert_eq!(obj["host"], "app.example.com"); + assert_eq!(obj["page_path"], "/checkout/123"); + assert_eq!( + obj["url"], + "https://App.Example.COM:8443/checkout/123?step=2#pay" + ); + + // Explicit page_path overrides the derived one. + let mut explicit = product_event(serde_json::json!({ + "name": "checkout_viewed", + "url": "https://app.example.com/checkout/123", + "page_path": "/checkout/:id", + })); + assert!(sanitize_product_event(&mut explicit, received_at())); + assert_eq!(explicit["page_path"], "/checkout/:id"); + assert_eq!(explicit["host"], "app.example.com"); + + // Relative URL: no host, path up to the query string. + let mut relative = + product_event(serde_json::json!({ "name": "e", "url": "/pricing?ref=x" })); + assert!(sanitize_product_event(&mut relative, received_at())); + assert_eq!(relative["host"], ""); + assert_eq!(relative["page_path"], "/pricing"); + + // No URL at all: both empty. + let mut none = product_event(serde_json::json!({ "name": "e" })); + assert!(sanitize_product_event(&mut none, received_at())); + assert_eq!(none["host"], ""); + assert_eq!(none["page_path"], ""); + assert_eq!(none["url"], ""); + } + + #[test] + fn product_event_row_shape_and_id_clamps() { + let mut obj = product_event(serde_json::json!({ + "name": "plan_started", + "user_id": "u".repeat(1000), + "group_id": "org_1", + "visitor_id": 123, + "service_name": "s".repeat(1000), + "org_id": "forged", + "seq": 7, + "kind": "navigation", + "unknown_field": true, + })); + assert!(sanitize_product_event(&mut obj, received_at())); + + let mut keys: Vec<&str> = obj.keys().map(String::as_str).collect(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "attributes", + "event_name", + "group_id", + "host", + "kind", + "page_path", + "seq", + "service_name", + "session_id", + "source", + "timestamp", + "url", + "user_id", + "visitor_id", + ] + ); + assert_eq!(obj["seq"], 0); + assert_eq!(obj["kind"], "custom"); + assert_eq!(obj["user_id"].as_str().unwrap().len(), PRODUCT_EVENT_MAX_ID_BYTES); + assert_eq!( + obj["service_name"].as_str().unwrap().len(), + PRODUCT_EVENT_MAX_SERVICE_NAME_BYTES + ); + // Off-type identifiers fall back to the column default. + assert_eq!(obj["visitor_id"], ""); + assert_eq!(obj["session_id"], ""); + assert!(obj["attributes"].as_object().unwrap().is_empty()); + } + + #[test] + fn product_event_attributes_are_clamped_like_session_events() { + let mut attributes = serde_json::Map::new(); + attributes.insert("a".repeat(500), serde_json::json!("v")); + for i in 0..100 { + attributes.insert(format!("k{i}"), serde_json::json!("v")); + } + attributes.insert("k0".to_string(), serde_json::json!("y".repeat(4096))); + attributes.insert("k1".to_string(), serde_json::json!(42)); + + let mut obj = product_event(serde_json::json!({ "name": "e" })); + obj.insert("attributes".to_string(), serde_json::Value::Object(attributes)); + assert!(sanitize_product_event(&mut obj, received_at())); + + let attributes = obj["attributes"].as_object().unwrap(); + assert_eq!(attributes.len(), SESSION_EVENT_MAX_ATTRIBUTES); + assert!(attributes.contains_key(&"a".repeat(SESSION_EVENT_MAX_ATTRIBUTE_KEY_BYTES))); + assert_eq!( + attributes["k0"].as_str().unwrap().len(), + SESSION_EVENT_MAX_ATTRIBUTE_VALUE_BYTES + ); + assert_eq!(attributes["k1"], "42"); + assert!(attributes.contains_key("k2")); + assert!(!attributes.contains_key("k99")); + + // Non-object attributes → empty map, row kept. + let mut off_type = product_event(serde_json::json!({ "name": "e", "attributes": "nope" })); + assert!(sanitize_product_event(&mut off_type, received_at())); + assert!(off_type["attributes"].as_object().unwrap().is_empty()); + } } diff --git a/apps/ingest/src/telemetry.rs b/apps/ingest/src/telemetry.rs index 5d8b09a20..640cde8be 100644 --- a/apps/ingest/src/telemetry.rs +++ b/apps/ingest/src/telemetry.rs @@ -331,6 +331,10 @@ pub enum TelemetrySignal { /// Carried separately from `SessionReplays` so per-signal metrics label it /// as `session_events` (matching its `maple.signal` span attribute). SessionEvents, + /// Product events posted directly by backends and mobile apps via + /// `POST /v1/events` (NDJSON, gateway-written). Lands in `product_events` + /// next to the browser rows the `session_events` MV materializes there. + ProductEvents, } impl TelemetrySignal { @@ -346,6 +350,7 @@ impl TelemetrySignal { Self::Metrics => "metrics", Self::SessionReplays => "session_replays", Self::SessionEvents => "session_events", + Self::ProductEvents => "product_events", } } } @@ -424,6 +429,7 @@ pub struct TinybirdConfig { pub datasource_session_replays: String, pub datasource_session_replay_events: String, pub datasource_session_events: String, + pub datasource_product_events: String, } impl TinybirdConfig { @@ -1472,6 +1478,7 @@ fn signal_tag(signal: TelemetrySignal) -> u8 { TelemetrySignal::Metrics => 3, TelemetrySignal::SessionReplays => 4, TelemetrySignal::SessionEvents => 5, + TelemetrySignal::ProductEvents => 6, } } @@ -1482,6 +1489,7 @@ fn signal_from_tag(tag: u8) -> Option { 3 => Some(TelemetrySignal::Metrics), 4 => Some(TelemetrySignal::SessionReplays), 5 => Some(TelemetrySignal::SessionEvents), + 6 => Some(TelemetrySignal::ProductEvents), _ => None, } } @@ -3026,6 +3034,7 @@ mod tests { datasource_session_replays: "session_replays".to_string(), datasource_session_replay_events: "session_replay_events".to_string(), datasource_session_events: "session_events".to_string(), + datasource_product_events: "product_events".to_string(), } } @@ -3982,6 +3991,7 @@ mod tests { assert_eq!(TelemetrySignal::Metrics.as_str(), "metrics"); assert_eq!(TelemetrySignal::SessionReplays.as_str(), "session_replays"); assert_eq!(TelemetrySignal::SessionEvents.as_str(), "session_events"); + assert_eq!(TelemetrySignal::ProductEvents.as_str(), "product_events"); } #[test] @@ -3994,6 +4004,7 @@ mod tests { TelemetrySignal::Metrics, TelemetrySignal::SessionReplays, TelemetrySignal::SessionEvents, + TelemetrySignal::ProductEvents, ] { assert_eq!(signal_from_tag(signal_tag(signal)), Some(signal)); } @@ -4793,6 +4804,7 @@ mod tests { "session_replays", "session_replay_events", "session_events", + "product_events", ] { let mapping = clickhouse_insert_mappings::mapping_for(datasource) .unwrap_or_else(|| panic!("missing ClickHouse mapping for {datasource}")); diff --git a/apps/landing/src/content/docs/session-replay/product-events-api.md b/apps/landing/src/content/docs/session-replay/product-events-api.md new file mode 100644 index 000000000..b09d76f01 --- /dev/null +++ b/apps/landing/src/content/docs/session-replay/product-events-api.md @@ -0,0 +1,68 @@ +--- +title: "Product events API" +description: "Post product events from a backend or mobile app to POST /v1/events on the Maple ingest gateway — the raw NDJSON contract every server-side track() call uses." +group: "Session Replay" +order: 3 +--- + +Browser page views and `track()` calls reach Maple through the session SDKs. Everything else — a +`signup_completed` from a webhook handler, a `plan_started` from your billing worker, a screen +view from a native app — is posted directly to the ingest gateway. Rows land in the same +`product_events` table as the browser events, so one funnel can span the marketing site, the app, +and the backend. + +## Endpoint + +``` +POST https://ingest.maple.dev/v1/events +Authorization: Bearer # or X-Maple-Ingest-Key: +Content-Type: application/x-ndjson +``` + +The body is **NDJSON**: one JSON object per line, any number of lines. The organization is +resolved from the ingest key — an `org_id` in the body is ignored. + +```json +{"name":"signup_completed","user_id":"user_01H…","service_name":"maple-api"} +{"name":"plan_started","user_id":"user_01H…","group_id":"org_01H…","attributes":{"plan":"startup"}} +{"name":"$screen","source":"mobile","visitor_id":"install-8f3…","page_path":"Checkout"} +``` + +## Fields + +| Field | Type | Notes | +| -------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | string | **Required.** 1–128 bytes. Names starting with `$` are reserved for Maple's SDKs and dropped, except `$screen` (mobile screen view, stored as `Kind = screen`). | +| `timestamp` | string | RFC 3339 (`2026-08-17T10:15:30.123Z`) or `YYYY-MM-DD HH:MM:SS[.fff]` (UTC). Defaults to the time the gateway received the batch. Stored as UTC. | +| `source` | string | `server` (default) or `mobile`. `browser` is reserved for the SDKs; other values drop the row. | +| `visitor_id` | string | Anonymous/device id — the browser SDK cookie value or a persistent mobile install id. ≤ 256 bytes. | +| `user_id` | string | Your user id after sign-in, matching what you pass to `identify()`. ≤ 256 bytes. | +| `group_id` | string | Account / workspace / org id. ≤ 256 bytes. | +| `session_id` | string | Optional link to a browser or mobile session. ≤ 256 bytes. | +| `service_name` | string | The emitting service (`maple-api`, `acme-ios`). ≤ 128 bytes. | +| `url` | string | Optional. `host` (lowercase) and `page_path` (pathname only) are derived from it. | +| `page_path` | string | Optional explicit path; overrides the one derived from `url`. Mobile `$screen` events put the screen name here. | +| `attributes` | object | Optional properties. ≤ 32 keys, key ≤ 64 bytes, value ≤ 1024 bytes; non-string values are stringified. | + +Over-long strings are truncated at the caps above; unknown fields are discarded. + +## Responses + +| Status | Meaning | +| ------ | ----------------------------------------------------------------------------------------------------------- | +| `200` | `{"accepted": }` — rows durably queued. Malformed rows (bad `name`, `source`, `timestamp`) are dropped individually and not counted. | +| `400` | A line is not valid JSON, or not a JSON object. The whole batch is rejected. | +| `401` | Missing or invalid ingest key. | +| `402` | The organization is out of quota for browser sessions (product events share that entitlement). | +| `503` | Storage temporarily unavailable — retry with backoff. | + +Product events are not metered separately: they are covered by the browser-sessions entitlement. + +## Example + +```bash +curl -X POST https://ingest.maple.dev/v1/events \ + -H "Authorization: Bearer $MAPLE_INGEST_KEY" \ + -H "Content-Type: application/x-ndjson" \ + --data-binary $'{"name":"plan_started","user_id":"user_123","attributes":{"plan":"startup"}}\n' +``` From 587ea58a02c774d69c8232ca08088247b743cb5a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:13:27 +0200 Subject: [PATCH 03/15] =?UTF-8?q?feat(cli):=20local=20schema=20v6=20?= =?UTF-8?q?=E2=80=94=20product=5Fevents=20+=20identity=5Flinks,=20drop=20w?= =?UTF-8?q?eb=5Fevents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v5→v6 store migration mirrors CH migration 0016: adds the session_events identity columns, creates product_events/identity_links (+MVs), backfills browser rows and identity pairs, drops web_events. --- apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + .../v5-to-v6-product-events.ts | 416 ++++ apps/cli/src/server/schema-identity.ts | 17 +- .../cli/src/server/schema/local-schema-v6.sql | 1773 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 2 +- apps/cli/test/local-store-migrations.test.ts | 199 +- apps/cli/test/native-local-store-migration.sh | 2 +- scripts/check-local-schema-manifest.ts | 15 + 10 files changed, 2419 insertions(+), 16 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v5-to-v6-product-events.ts create mode 100644 apps/cli/src/server/schema/local-schema-v6.sql diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 775481ca8..4660b9f83 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -58,4 +58,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "9d3b16f4f882049d40cf5bb31b9224243fcdade009c34b833212788f8cd9cc1d", projectRevision: "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7", }), + Object.freeze({ + version: 6, + fingerprint: "4fb7062f1e068837", + digest: "4fb7062f1e068837ff72848af8e862a47155b7543a1de8401b7b67c9ce176792", + manifestDigest: "c11f1f5aa250a7ce7eb42b0680197f4a69bdbfd0ada82dbfc95865010e78d7cb", + projectRevision: "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 40530abf3..9ea99d34f 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 5 as const +export const LOCAL_SCHEMA_VERSION = 6 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 8d3c1fc46..0cf1e855d 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -38,6 +38,7 @@ import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" import { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4-to-v5-service-overview-minutely" +import { v5ToV6ProductEventsModule } from "./local-store-migrations/v5-to-v6-product-events" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -122,6 +123,7 @@ export const localStoreMigrations: ReadonlyArray = v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, v4ToV5ServiceOverviewMinutelyModule, + v5ToV6ProductEventsModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-product-events.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-product-events.ts new file mode 100644 index 000000000..d38e0c19f --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-product-events.ts @@ -0,0 +1,416 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST, + LOCAL_SCHEMA_V6_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +const MODULE_ID = "local-0005-to-0006-product-events" as const + +/** + * Row counts the backfills must reproduce, captured on the v5 source before + * anything is rewritten and re-checked on the v6 target after. + * + * `browserEvents` is the number of `session_events` rows the projection admits; + * `identityPairs` is the number of distinct `(OrgId, VisitorId, UserId)` + * triples in `session_replays` — distinct because `identity_links` is a + * ReplacingMergeTree, so a raw count would depend on merge timing. + */ +interface V5ToV6SourceRows { + readonly browserEvents: string + readonly identityPairs: string +} + +interface V5ToV6State { + readonly module: typeof MODULE_ID + readonly version: 1 + readonly rawRows: Readonly> + readonly sourceRows: V5ToV6SourceRows + readonly retentionDays?: number +} + +interface V5ToV6Progress { + readonly installed: true +} + +/** + * The browser-row projection, byte-for-byte the SELECT of `product_events_mv` + * in the v6 snapshot and of `PRODUCT_EVENTS_PROJECTION_SQL` in migration 0016. + * A second copy is a second chance for a backfilled row and a live row of the + * same event to disagree; the verifier below pins the count, and the physical + * schema gate pins the view body, so drift between the two fails the migration + * rather than surfacing later as a page-view count that shifts at the backfill + * boundary. + */ +const PRODUCT_EVENTS_PROJECTION_SQL = `OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes` + +const PRODUCT_EVENTS_COLUMNS = + "OrgId, Timestamp, Source, SessionId, Seq, VisitorId, UserId, GroupId, Kind, EventName, Host, PagePath, Url, ServiceName, Attributes" + +const PRODUCT_EVENTS_SOURCE_FILTER = "Type IN ('navigation', 'custom')" + +const IDENTITY_LINKS_SOURCE_FILTER = "VisitorId != '' AND UserId != ''" + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const isCount = (value: unknown): value is string => typeof value === "string" && /^\d+$/.test(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v5 -> v6 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (!isCount(count)) throw new Error(`v5 -> v6 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v5 -> v6 rawRows contains an unknown table") + return counts +} + +const decodeSourceRows = (value: unknown): V5ToV6SourceRows => { + if (!isRecord(value)) throw new Error("v5 -> v6 sourceRows must be an object") + if (Object.keys(value).some((key) => key !== "browserEvents" && key !== "identityPairs")) + throw new Error("v5 -> v6 sourceRows contains an unknown field") + if (!isCount(value.browserEvents)) + throw new Error("v5 -> v6 sourceRows.browserEvents must be an unsigned decimal string") + if (!isCount(value.identityPairs)) + throw new Error("v5 -> v6 sourceRows.identityPairs must be an unsigned decimal string") + return { browserEvents: value.browserEvents, identityPairs: value.identityPairs } +} + +const decodeState = (value: unknown): V5ToV6State => { + if (!isRecord(value)) throw new Error("v5 -> v6 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "sourceRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v5 -> v6 state contains an unknown field") + if (value.module !== MODULE_ID || value.version !== 1) + throw new Error("v5 -> v6 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v5 -> v6 retentionDays must be an integer") + return { + module: MODULE_ID, + version: 1, + rawRows: decodeCounts(value.rawRows), + sourceRows: decodeSourceRows(value.sourceRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V5ToV6Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v5 -> v6 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const scalarCount = (db: Chdb, sql: string): string => { + const rows = parseJsonEachRow<{ count: string }>(db.query(sql)) + const count = rows[0]?.count + if (!isCount(count)) throw new Error(`v5 -> v6 count query returned no row: ${sql}`) + return count +} + +/** What the backfills must reproduce, measured on whichever side is open. */ +const sourceRowCounts = (db: Chdb): V5ToV6SourceRows => ({ + browserEvents: scalarCount( + db, + `SELECT toString(count()) AS count FROM session_events WHERE ${PRODUCT_EVENTS_SOURCE_FILTER}`, + ), + identityPairs: scalarCount( + db, + `SELECT toString(uniqExact(OrgId, VisitorId, UserId)) AS count FROM session_replays WHERE ${IDENTITY_LINKS_SOURCE_FILTER}`, + ), +}) + +const targetRowCounts = (db: Chdb): V5ToV6SourceRows => ({ + browserEvents: scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source = 'browser'", + ), + identityPairs: scalarCount( + db, + "SELECT toString(uniqExact(OrgId, VisitorId, UserId)) AS count FROM identity_links", + ), +}) + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V5_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const { rawRows, sourceRows } = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, retentionDays)) + return { rawRows: rawRowCounts(db), sourceRows: sourceRowCounts(db) } + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) + return { + module: MODULE_ID, + version: 1, + rawRows, + sourceRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * The local mirror of migration 0016, in its order: + * + * 1. `session_events` gains `VisitorId`/`UserId`/`GroupId` — metadata-only + * `ADD COLUMN IF NOT EXISTS`, no part is rewritten. The v6 bootstrap's + * `CREATE TABLE IF NOT EXISTS session_events` is a no-op on the cloned + * store, so the columns have to be added here or the physical gate fails. + * 2. The v6 snapshot bootstrap creates `product_events`, `identity_links` and + * their views. Views on `session_events`/`session_replays` are insert + * triggers on *those* tables, so their presence during the backfill below is + * inert — the `INSERT … SELECT` targets `product_events` directly. + * 3. Backfill. Unlike v3 -> v4 (which left `web_events` empty on purpose) the + * old table is being dropped, and `product_events` is the only place the + * 30-day `session_events` window can still be projected into; leaving it + * empty would lose the browser history the reader had yesterday. The + * idempotency step is 0016's `DELETE WHERE Source = 'browser'` — a + * lightweight delete, verified against the bundled chDB — rather than + * `TRUNCATE`: local mode has no direct `product_events` ingest yet, but the + * invariant is that a re-run of this step never destroys a row it cannot + * rebuild, and that stays true the day it does. `identity_links` needs no + * clear: a re-insert of a pair into a ReplacingMergeTree collapses on merge, + * and the verifier counts distinct triples for that reason. + * 4. `web_events_mv` then `web_events`, last, once the replacement is populated + * and its writer live. Every reader moved to `product_events` in the same + * release; the old table cannot be rebuilt past what the new one holds. + * + * Every statement is idempotent, so a resume after a crash between them lands + * in the same place. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("ALTER TABLE session_events ADD COLUMN IF NOT EXISTS VisitorId String DEFAULT ''") + db.exec("ALTER TABLE session_events ADD COLUMN IF NOT EXISTS UserId String DEFAULT ''") + db.exec("ALTER TABLE session_events ADD COLUMN IF NOT EXISTS GroupId String DEFAULT ''") + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) + return context.openTarget( + (db) => { + db.exec("DELETE FROM product_events WHERE Source = 'browser'") + db.exec( + `INSERT INTO product_events (${PRODUCT_EVENTS_COLUMNS}) SELECT ${PRODUCT_EVENTS_PROJECTION_SQL} FROM session_events WHERE ${PRODUCT_EVENTS_SOURCE_FILTER}`, + ) + db.exec( + `INSERT INTO identity_links (OrgId, VisitorId, UserId, FirstSeen) SELECT OrgId, VisitorId, UserId, StartTime AS FirstSeen FROM session_replays WHERE ${IDENTITY_LINKS_SOURCE_FILTER}`, + ) + db.exec("DROP VIEW IF EXISTS web_events_mv") + db.exec("DROP TABLE IF EXISTS web_events") + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: true }, + ) +} + +const verify = async ( + context: MigrationModuleContext, + state: V5ToV6State, + _progress: V5ToV6Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V6_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v5 -> v6 raw telemetry verification failed for ${table}`) + } + // The source is cloned, so re-measuring it on the target is the same + // number the preflight saw; comparing against the persisted state as + // well pins the resume path — a re-run cannot pass by projecting a + // source that changed under it. + const sourceRows = sourceRowCounts(db) + const backfilled = targetRowCounts(db) + for (const key of ["browserEvents", "identityPairs"] as const) { + if (sourceRows[key] !== state.sourceRows[key]) + throw new Error(`v5 -> v6 source ${key} changed between preflight and verify`) + if (backfilled[key] !== state.sourceRows[key]) + throw new Error( + `v5 -> v6 backfill verification failed for ${key}: expected ${state.sourceRows[key]}, found ${backfilled[key]}`, + ) + } + }, + { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v5-store", + description: "Clone the stopped v5 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-session-event-identity", + description: "Add the VisitorId, UserId and GroupId columns to session_events", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "install-product-events", + description: + "Install product_events and identity_links with their materialized views, backfill both from session_events and session_replays, then drop web_events", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v6-schema", + description: + "Verify the v6 physical schema, retained raw telemetry counts, and the backfilled row counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v5 store is cloned byte-for-byte before any DDL runs.", + }, + { + name: "session_events", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Three columns are added as metadata-only defaults; no part is rewritten and every existing row reads back unchanged with '' in the new columns.", + }, + { + name: "session_replays", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "Read once to seed identity_links; neither rewritten nor re-keyed.", + }, + { + // The browser half of product_events is rebuilt in full from the source + // window that still exists: session_events keeps 30 days, so the new + // table starts with exactly what web_events could have held. It is + // verified row-for-row against the source count. Rows written to + // product_events directly, once local ingest carries them, have no + // source and are never touched by the browser-only clear. + name: "product_events", + classification: "derived", + disposition: "rebuild-complete", + guarantee: + "Browser rows are projected from every retained session_events row and the count is verified; the projection is the view body, so backfilled and live rows agree.", + preservationInterval: "session_events retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, + { + name: "identity_links", + classification: "derived", + disposition: "rebuild-complete", + guarantee: + "Every identified (VisitorId, UserId) pair in retained session_replays is linked; distinct-pair count verified against the source.", + preservationInterval: "session_replays retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, + { + // Dropped, not migrated: product_events supersedes it and holds a + // superset of what it could contain (same source, same window, wider + // projection). Its rows were derived from session_events, which is + // preserved, so nothing authoritative leaves the store. + name: "web_events", + classification: "derived", + disposition: "invalidate", + guarantee: + "Replaced by product_events, which is backfilled from the same session_events window before web_events is dropped; every reader moved in the same release.", + }, +] + +export const v5ToV6ProductEventsModule: LocalStoreMigrationModule = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Add identity columns to session_events; replace web_events with the backfilled product_events table and add identity_links", + from: LOCAL_SCHEMA_V5, + to: LOCAL_SCHEMA_V6, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index aedf4d40a..f7cdde738 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -4,6 +4,7 @@ import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } +import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -27,7 +28,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" + "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -63,6 +64,11 @@ export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest +/** Immutable v6 DDL/manifest snapshot used by the v5 -> v6 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V6_SQL = schemaV6Sql +export const LOCAL_SCHEMA_V6_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV6Sql) +export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -122,6 +128,15 @@ export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, }) +export const LOCAL_SCHEMA_V6: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[6]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[6]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[6]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[6]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[6]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-schema-v6.sql b/apps/cli/src/server/schema/local-schema-v6.sql new file mode 100644 index 000000000..5745a92ef --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v6.sql @@ -0,0 +1,1773 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04 +-- localSchemaVersion: 6 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen DateTime64(9) +) +ENGINE = ReplacingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 38c6a1735..5745a92ef 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. -- projectRevision: bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04 --- localSchemaVersion: 5 +-- localSchemaVersion: 6 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 6b0211435..6fff493fa 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -14,6 +14,8 @@ import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V6, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -50,21 +52,22 @@ import { duplicateCursorContinuation, type CopyProgress, } from "../src/server/local-store-migrations/legacy-to-current" +import { v5ToV6ProductEventsModule } from "../src/server/local-store-migrations/v5-to-v6-product-events" import { mkdir, mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("c36c52a95568eb68") - expect(SCHEMA_DIGEST).toBe("c36c52a95568eb68f8ebc98d7d36b552f21fb09b888bb310c68f0ad52d529fe4") + it("matches the generated v6 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("4fb7062f1e068837") + expect(SCHEMA_DIGEST).toBe("4fb7062f1e068837ff72848af8e862a47155b7543a1de8401b7b67c9ce176792") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(6) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V6) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -89,12 +92,15 @@ describe("current local schema identity", () => { // v4 is exactly v3 plus the web analytics fact table and its view. Asserted // against the frozen v3 manifest rather than the diff so a later structural - // change can't quietly ride along on this version. - const webEvents = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "web_events") + // change can't quietly ride along on this version. (v6 drops web_events + // again, so it is read from the frozen v4 manifest, not the current one.) + const webEvents = LOCAL_SCHEMA_V4_MANIFEST.objects.find((object) => object.name === "web_events") expect(webEvents?.engine).toBe("MergeTree") expect(webEvents?.orderBy).toBe("(OrgId, Timestamp, SessionId, Seq)") expect(webEvents?.indexes).toContain("idx_event_name") - const webEventsView = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "web_events_mv") + const webEventsView = LOCAL_SCHEMA_V4_MANIFEST.objects.find( + (object) => object.name === "web_events_mv", + ) expect(webEventsView?.definition).toContain("FROM session_events") const v3Names = new Set(LOCAL_SCHEMA_V3_MANIFEST.objects.map((object) => object.name)) expect(v3Names.has("web_events")).toBe(false) @@ -104,14 +110,14 @@ describe("current local schema identity", () => { // v5 is exactly v4 plus the minutely service-overview rollup and its view. // Asserted against the frozen v4 manifest, same as above, so a later // structural change cannot quietly ride along on this version. - const minutely = LOCAL_SCHEMA_MANIFEST.objects.find( + const minutely = LOCAL_SCHEMA_V5_MANIFEST.objects.find( (object) => object.name === "service_overview_minutely", ) expect(minutely?.engine).toBe("AggregatingMergeTree") expect(minutely?.orderBy).toBe( "(OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha)", ) - const minutelyView = LOCAL_SCHEMA_MANIFEST.objects.find( + const minutelyView = LOCAL_SCHEMA_V5_MANIFEST.objects.find( (object) => object.name === "service_overview_minutely_mv", ) // Reads traces directly — a cascade off the hourly rollup would make the @@ -119,8 +125,67 @@ describe("current local schema identity", () => { expect(minutelyView?.definition).toContain("FROM traces") expect(minutelyView?.definition).not.toContain("FROM service_overview_minutely") expect( - LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v4Names.has(name)), + LOCAL_SCHEMA_V5_MANIFEST.objects + .map((object) => object.name) + .filter((name) => !v4Names.has(name)), ).toEqual(["service_overview_minutely", "service_overview_minutely_mv"]) + + // v6 replaces web_events with product_events, adds identity_links, and + // widens session_events by the three identity columns. Asserted against + // the frozen v5 manifest, same as above. + const productEvents = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "product_events") + expect(productEvents?.engine).toBe("MergeTree") + expect(productEvents?.orderBy).toBe("(OrgId, Timestamp, VisitorId, SessionId, Seq)") + expect(productEvents?.ttl).toContain("365 DAY") + expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id"]) + expect(productEvents?.columns.map((column) => column.name)).toEqual([ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + ]) + const productEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( + (object) => object.name === "product_events_mv", + ) + expect(productEventsView?.definition).toContain("FROM session_events") + expect(productEventsView?.definition).toContain("'browser' AS Source") + const identityLinks = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "identity_links") + expect(identityLinks?.engine).toBe("ReplacingMergeTree") + expect(identityLinks?.orderBy).toBe("(OrgId, VisitorId, UserId)") + const identityLinksView = LOCAL_SCHEMA_MANIFEST.objects.find( + (object) => object.name === "identity_links_mv", + ) + expect(identityLinksView?.definition).toContain("FROM session_replays") + const sessionEventColumns = (manifest: LocalSchemaManifest) => + manifest.objects + .find((object) => object.name === "session_events") + ?.columns.map((column) => column.name) ?? [] + expect(sessionEventColumns(LOCAL_SCHEMA_V5_MANIFEST)).not.toContain("VisitorId") + expect(sessionEventColumns(LOCAL_SCHEMA_MANIFEST).slice(-3)).toEqual([ + "VisitorId", + "UserId", + "GroupId", + ]) + const v5Names = new Set(LOCAL_SCHEMA_V5_MANIFEST.objects.map((object) => object.name)) + const v6Names = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) + expect([...v6Names].filter((name) => !v5Names.has(name))).toEqual([ + "identity_links", + "identity_links_mv", + "product_events", + "product_events_mv", + ]) + expect([...v5Names].filter((name) => !v6Names.has(name))).toEqual(["web_events", "web_events_mv"]) }) }) @@ -133,6 +198,7 @@ describe("local migration registry", () => { "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", "local-0004-to-0005-service-overview-minutely", + "local-0005-to-0006-product-events", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -140,6 +206,7 @@ describe("local migration registry", () => { expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) + expect(chain[5]?.to).toEqual(LOCAL_SCHEMA_V6) expect(typeof chain[0]?.apply).toBe("function") }) @@ -178,7 +245,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 7, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -970,3 +1037,111 @@ describe("legacy raw replay cursor", () => { expect(() => legacyToCurrentModule.decodeProgress(progress)).toThrow(/lastHash/) }) }) + +describe("v5 -> v6 product events module", () => { + const rawRows = { + logs: "1", + traces: "2", + metrics_sum: "0", + metrics_gauge: "0", + metrics_histogram: "0", + metrics_exponential_histogram: "0", + } + const state = { + module: "local-0005-to-0006-product-events", + version: 1, + rawRows, + sourceRows: { browserEvents: "3", identityPairs: "2" }, + } + + it("binds the frozen v5 and v6 identities and never the current constant", () => { + expect(v5ToV6ProductEventsModule.from).toEqual(LOCAL_SCHEMA_V5) + expect(v5ToV6ProductEventsModule.to).toEqual(LOCAL_SCHEMA_V6) + expect(v5ToV6ProductEventsModule.from).not.toBe(CURRENT_LOCAL_SCHEMA) + expect(v5ToV6ProductEventsModule.operations.map((operation) => operation.id)).toEqual([ + "clone-v5-store", + "add-session-event-identity", + "install-product-events", + "verify-v6-schema", + ]) + // The chain must reach v6 through this module and only this module. + const chain = resolveMigrationChain(LOCAL_SCHEMA_V5, CURRENT_LOCAL_SCHEMA) + expect(chain.map((migration) => migration.id)).toEqual(["local-0005-to-0006-product-events"]) + expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V6) + // The dropped table is declared, and the backfilled ones say what they + // are rebuilt from. + const dispositions = new Map( + v5ToV6ProductEventsModule.dispositions.map((entry) => [entry.name, entry.disposition]), + ) + expect(dispositions.get("web_events")).toBe("invalidate") + expect(dispositions.get("product_events")).toBe("rebuild-complete") + expect(dispositions.get("identity_links")).toBe("rebuild-complete") + expect(dispositions.get("session_events")).toBe("preserve-exact") + }) + + it("decodes only its own well-formed persisted state", () => { + expect(v5ToV6ProductEventsModule.decodeState(state)).toEqual(state) + expect(v5ToV6ProductEventsModule.decodeState({ ...state, retentionDays: 120 })).toEqual({ + ...state, + retentionDays: 120, + }) + expect(() => + v5ToV6ProductEventsModule.decodeState({ + ...state, + module: "local-0004-to-0005-service-overview-minutely", + }), + ).toThrow(/unsupported module or version/) + expect(() => v5ToV6ProductEventsModule.decodeState({ ...state, version: 2 })).toThrow( + /unsupported module or version/, + ) + expect(() => v5ToV6ProductEventsModule.decodeState({ ...state, extra: true })).toThrow( + /unknown field/, + ) + expect(() => v5ToV6ProductEventsModule.decodeState({ ...state, retentionDays: "120" })).toThrow( + /retentionDays must be an integer/, + ) + expect(() => + v5ToV6ProductEventsModule.decodeState({ ...state, rawRows: { ...rawRows, logs: "-1" } }), + ).toThrow(/rawRows.logs/) + expect(() => + v5ToV6ProductEventsModule.decodeState({ ...state, rawRows: { ...rawRows, web_events: "1" } }), + ).toThrow(/unknown table/) + // The backfill counts are part of the resume key: a state without them + // cannot verify, so it must not decode. + const { sourceRows: _sourceRows, ...withoutSourceRows } = state + expect(() => v5ToV6ProductEventsModule.decodeState(withoutSourceRows)).toThrow( + /sourceRows must be an object/, + ) + expect(() => + v5ToV6ProductEventsModule.decodeState({ ...state, sourceRows: { browserEvents: "3" } }), + ).toThrow(/identityPairs/) + expect(() => + v5ToV6ProductEventsModule.decodeState({ + ...state, + sourceRows: { browserEvents: 3, identityPairs: "2" }, + }), + ).toThrow(/browserEvents/) + expect(() => + v5ToV6ProductEventsModule.decodeState({ + ...state, + sourceRows: { browserEvents: "3", identityPairs: "2", webEvents: "0" }, + }), + ).toThrow(/unknown field/) + }) + + it("decodes progress as the single installed marker", () => { + expect(v5ToV6ProductEventsModule.decodeProgress(undefined)).toBeUndefined() + expect(v5ToV6ProductEventsModule.decodeProgress({ installed: true })).toEqual({ installed: true }) + expect(() => v5ToV6ProductEventsModule.decodeProgress({ installed: false })).toThrow(/invalid/) + expect(() => v5ToV6ProductEventsModule.decodeProgress({ installed: true, rows: 1 })).toThrow( + /invalid/, + ) + }) + + it("recovers by keeping whatever state and progress were persisted", async () => { + const progress = { installed: true } as const + await expect( + v5ToV6ProductEventsModule.recover({} as MigrationModuleContext, state as never, progress), + ).resolves.toEqual({ state, progress }) + }) +}) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 8c61ab35f..a667b54fb 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 5 and .schema == "c36c52a95568eb68"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 6 and .schema == "4fb7062f1e068837"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index 328c60c6a..644acbc02 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -19,6 +19,9 @@ import { LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST_DIGEST, LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST_DIGEST, + LOCAL_SCHEMA_V6_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -119,6 +122,18 @@ if ( fail("the immutable local schema v5 snapshot no longer matches its historical identity") } +const v6 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V6.version) +if ( + !v6 || + LOCAL_SCHEMA_V6_MANIFEST_DIGEST !== v6.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V6_SQL) !== v6.fingerprint || + schemaDigest(LOCAL_SCHEMA_V6_SQL) !== v6.digest || + LOCAL_SCHEMA_V6.fingerprint !== v6.fingerprint || + LOCAL_SCHEMA_V6.digest !== v6.digest +) { + fail("the immutable local schema v6 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names") From dfa3e2e7421d5277906fc4806d48abddecdc1396 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:14:44 +0200 Subject: [PATCH 04/15] feat(sdk): stamp identity on session events; server-side MapleEvents.track() Browser/effect-sdk sinks emit visitor_id/user_id/group_id on every session event so product_events carries the person key without a join. effect-sdk server entry gains MapleEvents (batched NDJSON POST /v1/events, never throws to the caller). --- .../src/content/docs/sdks/effect-server.md | 30 ++ .../docs/session-replay/browser-sdk.md | 8 + packages/browser-session/package.json | 3 +- .../src/events/events-sink.test.ts | 62 ++++ .../browser-session/src/events/events-sink.ts | 24 +- packages/browser-session/src/events/props.ts | 65 ++++ packages/browser-session/src/events/track.ts | 66 +--- .../browser-session/src/platform/transport.ts | 15 + .../src/session/replay-session.ts | 1 + packages/browser/src/init.ts | 1 + .../effect-sdk/src/client/replay-loader.ts | 1 + packages/effect-sdk/src/server/events.test.ts | 250 +++++++++++++++ packages/effect-sdk/src/server/events.ts | 299 ++++++++++++++++++ packages/effect-sdk/src/server/index.ts | 2 + 14 files changed, 761 insertions(+), 66 deletions(-) create mode 100644 packages/browser-session/src/events/props.ts create mode 100644 packages/effect-sdk/src/server/events.test.ts create mode 100644 packages/effect-sdk/src/server/events.ts diff --git a/apps/landing/src/content/docs/sdks/effect-server.md b/apps/landing/src/content/docs/sdks/effect-server.md index a1a628e93..5b73d07e1 100644 --- a/apps/landing/src/content/docs/sdks/effect-server.md +++ b/apps/landing/src/content/docs/sdks/effect-server.md @@ -84,6 +84,36 @@ Managed platforms expose the commit SHA automatically. The **environment** is on For self-hosted deployments, set `COMMIT_SHA` in your build pipeline and `MAPLE_ENVIRONMENT` at runtime. +## Server-side `track()` + +Some funnel steps only happen on the backend — a `signup_completed` in a webhook handler, a +`plan_started` when billing confirms a subscription. `MapleEvents` posts those to Maple's +[product events endpoint](/docs/session-replay/product-events-api) so they land in the same +`product_events` table as the browser SDK's `track()` calls, keyed to the same person. + +```typescript +import { MapleEvents } from "@maple-dev/effect-sdk/server" +import { Effect, Layer } from "effect" + +const EventsLive = MapleEvents.layer({ serviceName: "billing" }) + +const onSubscriptionCreated = Effect.fn("onSubscriptionCreated")(function* (userId: string, orgId: string, plan: string) { + const events = yield* MapleEvents.MapleEvents + yield* events.track("plan_started", { userId, groupId: orgId, attributes: { plan } }) +}) +``` + +`track(name, options)` buffers the event and returns immediately; batches go out every 5 seconds, +at 100 events, and when the layer's scope closes. It never fails the caller — a rejected batch is +dropped with a rate-limited console warning. `options` are all optional: `userId`, `groupId`, +`visitorId` (the browser cookie value, if your backend has it), `sessionId`, `timestamp`, `url`, +`pagePath`, and `attributes` (coerced and capped exactly like the browser `track()`). + +The endpoint and ingest key resolve the same way as the tracer (`endpoint`/`ingestKey` in config, +else `MAPLE_ENDPOINT` / `MAPLE_INGEST_KEY`); without a key, events are dropped with a one-shot +warning. Outside an Effect runtime, `MapleEvents.makeHandle(config)` returns a plain +`{ track, flush, dispose }` — call `dispose()` on shutdown so the last batch is sent. + ## Verify 1. Start your application. diff --git a/apps/landing/src/content/docs/session-replay/browser-sdk.md b/apps/landing/src/content/docs/session-replay/browser-sdk.md index d7fe8d98f..ff97ab875 100644 --- a/apps/landing/src/content/docs/session-replay/browser-sdk.md +++ b/apps/landing/src/content/docs/session-replay/browser-sdk.md @@ -185,6 +185,14 @@ Names are capped at 128 chars; props at 32 keys / 64-char keys / 1024-char value Values are coerced to strings (`Date` → ISO, objects → JSON; `null`/`undefined`/functions are dropped). Calls before `init()` finishes are queued, and `track()` never throws. +Every session event — page views and `track()` calls alike — is stamped with the person it belongs +to: the visitor id, plus the `id` and `groupId` from `identify()`. That is what lets a funnel follow +one person from an anonymous marketing visit through sign-in, and lets browser events line up with +the same user's [server-side events](/docs/sdks/effect-server#server-side-track). Identity is +resolved when the batch is sent, so an `identify()` shortly after `init()` still lands on the first +page view. The visitor id is empty when the visitor cookie is off (consent not granted, Global +Privacy Control, `persistVisitorId: false`); events from older SDK builds arrive with no identity. + ## Linking a marketing site to your app The visitor id is stored in **both** localStorage and a cookie scoped to your registered domain, so diff --git a/packages/browser-session/package.json b/packages/browser-session/package.json index 0b964fe90..d0cb25b76 100644 --- a/packages/browser-session/package.json +++ b/packages/browser-session/package.json @@ -7,7 +7,8 @@ "main": "./src/index.ts", "exports": { ".": "./src/index.ts", - "./replay": "./src/session/replay-session.ts" + "./replay": "./src/session/replay-session.ts", + "./props": "./src/events/props.ts" }, "scripts": { "typecheck": "tsc --noEmit", diff --git a/packages/browser-session/src/events/events-sink.test.ts b/packages/browser-session/src/events/events-sink.test.ts index 8fb93ae43..86dbfd75b 100644 --- a/packages/browser-session/src/events/events-sink.test.ts +++ b/packages/browser-session/src/events/events-sink.test.ts @@ -9,6 +9,8 @@ vi.mock("../session/session", () => ({ vi.mock("../platform/transport", () => ({ postSessionEvents: vi.fn(async () => {}) })) const { resetSinkForTests, startEventSink } = await import("./events-sink") +const { postSessionEvents } = await import("../platform/transport") +const { resetVisitorCacheForTests, setVisitorTracking } = await import("../identity/visitor") const CONFIG = { endpoint: "https://ingest.test", @@ -79,3 +81,63 @@ describe("startEventSink baseline counters", () => { expect(sink.getClickCount()).toBe(0) }) }) + +// Every row carries the person key funnels group on. Resolved when the row is +// built, so a late `identify()` still lands on the buffered page view. +describe("startEventSink identity stamping", () => { + const postedRows = () => vi.mocked(postSessionEvents).mock.calls.flatMap(([, rows]) => rows) + + beforeEach(() => { + resetSinkForTests() + resetVisitorCacheForTests() + window.localStorage.clear() + vi.mocked(postSessionEvents).mockClear() + }) + + it("stamps visitor_id, user_id and group_id on every row", async () => { + const sink = startEventSink( + { ...CONFIG, getIdentity: () => ({ id: "user_1", groupId: "org_1", traits: {} }) }, + "sess-id-1", + ) + sink.emit({ type: "custom", message: "signup_completed" }) + await sink.flush() + + const rows = postedRows() + expect(rows.length).toBeGreaterThan(0) + for (const row of rows) { + expect(row.session_id).toBe("sess-id-1") + expect(row.user_id).toBe("user_1") + expect(row.group_id).toBe("org_1") + expect(typeof row.visitor_id).toBe("string") + expect(row.visitor_id).not.toBe("") + } + sink.stop() + }) + + it("reads identity when the row is built, so identify() after emit still applies", async () => { + let identity: { id: string; traits: Record } | undefined + const sink = startEventSink({ ...CONFIG, getIdentity: () => identity }, "sess-id-2") + sink.emit({ type: "custom", message: "before_identify" }) + identity = { id: "user_late", traits: {} } + await sink.flush() + + expect(postedRows().every((row) => row.user_id === "user_late")).toBe(true) + sink.stop() + }) + + it("sends empty strings when there is no identity and visitor tracking is off", async () => { + setVisitorTracking(false) + const sink = startEventSink(CONFIG, "sess-id-3") + sink.emit({ type: "custom", message: "anon" }) + await sink.flush() + + const rows = postedRows() + expect(rows.length).toBeGreaterThan(0) + for (const row of rows) { + expect(row.visitor_id).toBe("") + expect(row.user_id).toBe("") + expect(row.group_id).toBe("") + } + sink.stop() + }) +}) diff --git a/packages/browser-session/src/events/events-sink.ts b/packages/browser-session/src/events/events-sink.ts index a85d9eebc..bb3e354ce 100644 --- a/packages/browser-session/src/events/events-sink.ts +++ b/packages/browser-session/src/events/events-sink.ts @@ -5,6 +5,7 @@ import { postSessionEvents } from "../platform/transport" import { approximateSize } from "../platform/approximate-size" import { markActivity, noteNavigation } from "../session/session" import { activeTraceId } from "./trace-id" +import { getVisitorId } from "../identity/visitor" /** * A distilled, structured session event. Sparse: only the fields relevant to @@ -114,7 +115,7 @@ export function startEventSink(config: IngestConfig, sessionId: string): Session const batch = buffer buffer = [] bufferBytes = 0 - const rows = batch.map(({ ev, seq }) => toRow(sessionId, ev, seq)) + const rows = batch.map(({ ev, seq }) => toRow(config, sessionId, ev, seq)) await postSessionEvents(config, rows, keepalive) } @@ -287,10 +288,27 @@ function drainPending(sink: SessionEventSink): void { for (const ev of queued) sink.emit(ev) } -/** Map an internal event to the snake_case ingest row (org_id is added server-side). */ -function toRow(sessionId: string, ev: SessionEvent, seq: number): Record { +/** + * Map an internal event to the snake_case ingest row (org_id is added server-side). + * + * Identity is resolved here, at flush time, alongside the trace id: it is the + * person key funnels group on, and reading it late means an `identify()` that + * lands shortly after init still stamps the initial page view. `visitor_id` is + * `""` whenever the visitor cookie is off (consent, GPC, `persistVisitorId: + * false`) — the same rule the metadata row applies. + */ +function toRow( + config: IngestConfig, + sessionId: string, + ev: SessionEvent, + seq: number, +): Record { + const identity = config.getIdentity?.() return { session_id: sessionId, + visitor_id: getVisitorId() ?? "", + user_id: identity?.id ?? "", + group_id: identity?.groupId ?? "", timestamp: formatCHDateTime(new Date(ev.timestamp ?? Date.now())), seq, type: ev.type, diff --git a/packages/browser-session/src/events/props.ts b/packages/browser-session/src/events/props.ts new file mode 100644 index 000000000..a1091e277 --- /dev/null +++ b/packages/browser-session/src/events/props.ts @@ -0,0 +1,65 @@ +// Property coercion for `track()`, shared by the browser sink and the server +// SDK's product-events client. Pure: no browser globals, so it can be bundled +// into a Node entrypoint without dragging the session engine along. + +/** Properties a host app may attach to a custom event. */ +export type TrackProps = Readonly> + +/** + * Caps mirroring what the ingest gateway enforces. Applying them here too means + * an over-sized event is trimmed before it costs bandwidth, and the developer + * sees the same shape locally that the warehouse will store. + */ +export const MAX_EVENT_NAME_LENGTH = 128 +const MAX_PROPS = 32 +const MAX_PROP_KEY_LENGTH = 64 +const MAX_PROP_VALUE_LENGTH = 1024 +const MAX_TOTAL_PROP_BYTES = 8 * 1024 + +/** + * Coerce one property value to the string the warehouse column holds. + * + * `null`/`undefined`/functions/symbols are dropped rather than stringified — + * `"undefined"` as a stored value is worse than an absent key. + */ +function coerce(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + switch (typeof value) { + case "string": + return value.slice(0, MAX_PROP_VALUE_LENGTH) + case "number": + case "boolean": + case "bigint": + return String(value) + case "function": + case "symbol": + return undefined + default: + break + } + try { + if (value instanceof Date) return value.toISOString() + return JSON.stringify(value)?.slice(0, MAX_PROP_VALUE_LENGTH) + } catch { + // Circular structure — drop the key rather than throw into the caller. + return undefined + } +} + +/** Coerce and cap a `track()` property bag to the `Map(String, String)` the warehouse stores. */ +export function coerceTrackProps(props: TrackProps | undefined): Record { + if (!props) return {} + const out: Record = {} + let bytes = 0 + for (const [rawKey, rawValue] of Object.entries(props)) { + if (Object.keys(out).length >= MAX_PROPS) break + const value = coerce(rawValue) + if (value === undefined) continue + const key = rawKey.slice(0, MAX_PROP_KEY_LENGTH) + if (!key) continue + bytes += key.length + value.length + if (bytes > MAX_TOTAL_PROP_BYTES) break + out[key] = value + } + return out +} diff --git a/packages/browser-session/src/events/track.ts b/packages/browser-session/src/events/track.ts index e8479acaf..238ea59bc 100644 --- a/packages/browser-session/src/events/track.ts +++ b/packages/browser-session/src/events/track.ts @@ -1,69 +1,11 @@ import { hasConsent } from "../identity/consent" import { getActiveSink, queuePending, type SessionEvent } from "./events-sink" +import { coerceTrackProps, MAX_EVENT_NAME_LENGTH, type TrackProps } from "./props" -/** Properties a host app may attach to a custom event. */ -export type TrackProps = Readonly> - -/** - * Caps mirroring what the ingest gateway enforces. Applying them here too means - * an over-sized event is trimmed before it costs bandwidth, and the developer - * sees the same shape locally that the warehouse will store. - */ -const MAX_NAME_LENGTH = 128 -const MAX_PROPS = 32 -const MAX_PROP_KEY_LENGTH = 64 -const MAX_PROP_VALUE_LENGTH = 1024 -const MAX_TOTAL_PROP_BYTES = 8 * 1024 +export type { TrackProps } from "./props" let warnedAboutName = false -/** - * Coerce one property value to the string the warehouse column holds. - * - * `null`/`undefined`/functions/symbols are dropped rather than stringified — - * `"undefined"` as a stored value is worse than an absent key. - */ -function coerce(value: unknown): string | undefined { - if (value === null || value === undefined) return undefined - switch (typeof value) { - case "string": - return value.slice(0, MAX_PROP_VALUE_LENGTH) - case "number": - case "boolean": - case "bigint": - return String(value) - case "function": - case "symbol": - return undefined - default: - break - } - try { - if (value instanceof Date) return value.toISOString() - return JSON.stringify(value)?.slice(0, MAX_PROP_VALUE_LENGTH) - } catch { - // Circular structure — drop the key rather than throw into the caller. - return undefined - } -} - -function coerceProps(props: TrackProps | undefined): Record { - if (!props) return {} - const out: Record = {} - let bytes = 0 - for (const [rawKey, rawValue] of Object.entries(props)) { - if (Object.keys(out).length >= MAX_PROPS) break - const value = coerce(rawValue) - if (value === undefined) continue - const key = rawKey.slice(0, MAX_PROP_KEY_LENGTH) - if (!key) continue - bytes += key.length + value.length - if (bytes > MAX_TOTAL_PROP_BYTES) break - out[key] = value - } - return out -} - /** * Record a custom product event against the current session. * @@ -86,8 +28,8 @@ export function track(name: string, props?: TrackProps): void { const ev: SessionEvent = { type: "custom", - message: name.trim().slice(0, MAX_NAME_LENGTH), - attrs: coerceProps(props), + message: name.trim().slice(0, MAX_EVENT_NAME_LENGTH), + attrs: coerceTrackProps(props), timestamp: Date.now(), // Captured now rather than at flush time so a queued event reports the // page it actually happened on. diff --git a/packages/browser-session/src/platform/transport.ts b/packages/browser-session/src/platform/transport.ts index 0abf31619..8c77e55db 100644 --- a/packages/browser-session/src/platform/transport.ts +++ b/packages/browser-session/src/platform/transport.ts @@ -7,6 +7,21 @@ export interface IngestConfig { readonly ingestKey: string readonly maskAllInputs: boolean readonly maskAllText: boolean + /** + * Identity from `identify()`, consulted when session-event rows are built so + * a late `identify()` still stamps `user_id`/`group_id` on the rows that + * follow it — the same source the session metadata row reads. + */ + readonly getIdentity?: (() => EventIdentity | undefined) | undefined +} + +/** + * The slice of `ResolvedIdentity` that rides on session-event rows. Declared + * structurally so this transport layer stays free of the identity module. + */ +export interface EventIdentity { + readonly id?: string | undefined + readonly groupId?: string | undefined } // Replay POSTs are best-effort and must never throw into the host app, but a diff --git a/packages/browser-session/src/session/replay-session.ts b/packages/browser-session/src/session/replay-session.ts index 5594b2d92..4703ea653 100644 --- a/packages/browser-session/src/session/replay-session.ts +++ b/packages/browser-session/src/session/replay-session.ts @@ -44,6 +44,7 @@ export function startReplaySession(options: ReplaySessionOptions): ReplaySession ingestKey: options.ingestKey, maskAllInputs: options.maskAllInputs, maskAllText: options.maskAllText, + getIdentity: options.getIdentity, } let recorder: Recorder | undefined diff --git a/packages/browser/src/init.ts b/packages/browser/src/init.ts index 965d5ec28..ba05dbed8 100644 --- a/packages/browser/src/init.ts +++ b/packages/browser/src/init.ts @@ -85,6 +85,7 @@ export function init(rawConfig: MapleBrowserConfig): MapleBrowserHandle { ingestKey: config.ingestKey, maskAllInputs: config.maskAllInputs, maskAllText: config.maskAllText, + getIdentity: () => activeConfig?.identity, }, session.id, ) diff --git a/packages/effect-sdk/src/client/replay-loader.ts b/packages/effect-sdk/src/client/replay-loader.ts index 002c742e2..8945a8f03 100644 --- a/packages/effect-sdk/src/client/replay-loader.ts +++ b/packages/effect-sdk/src/client/replay-loader.ts @@ -76,6 +76,7 @@ export const startClientSession = (config: ClientSessionConfig): ClientSessionHa ingestKey: config.ingestKey, maskAllInputs: config.replay?.maskAllInputs ?? true, maskAllText: config.replay?.maskAllText ?? false, + getIdentity: getCurrentIdentity, } const replayEnabled = (config.replay?.enabled ?? true) && typeof document !== "undefined" const sampled = replayEnabled && Math.random() < (config.replay?.sampleRate ?? 1) diff --git a/packages/effect-sdk/src/server/events.test.ts b/packages/effect-sdk/src/server/events.test.ts new file mode 100644 index 000000000..0656f3d70 --- /dev/null +++ b/packages/effect-sdk/src/server/events.test.ts @@ -0,0 +1,250 @@ +import { describe, it } from "@effect/vitest" +import { Effect, Layer, Schema, Scope } from "effect" +import { HttpClient, HttpClientResponse } from "effect/unstable/http" +import { afterEach, expect, vi } from "vitest" +import { make, makeHandle, MapleEvents } from "./events.js" + +interface PostedBatch { + readonly url: string + readonly headers: Record + readonly contentType: string | undefined + readonly lines: Array> +} + +/** One NDJSON line as the gateway would parse it. */ +const parseLine = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +) + +/** An `HttpClient` that records every NDJSON POST and answers with `status`. */ +const stubClient = (status = 200) => { + const posts: Array = [] + const client = HttpClient.make((request) => { + const body = request.body + const text = body._tag === "Uint8Array" ? new TextDecoder().decode(body.body) : "" + posts.push({ + url: request.url, + headers: { ...request.headers }, + contentType: body._tag === "Uint8Array" ? body.contentType : undefined, + lines: text + .split("\n") + .filter((line) => line.length > 0) + .map((line) => parseLine(line)), + }) + return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status }))) + }) + return { posts, layer: Layer.succeed(HttpClient.HttpClient, client) } +} + +// Explicit config so nothing depends on ambient env; interval off so tests +// drive flushes themselves. +const config = { + serviceName: "billing", + endpoint: "https://ingest.test/", + ingestKey: "secret", + flushInterval: 0, +} as const + +const withEvents = ( + cfg: Parameters[0], + httpLayer: Layer.Layer, + body: (events: MapleEvents["Service"]) => Effect.Effect, +) => + Effect.scoped( + Effect.gen(function* () { + const events = yield* make(cfg).pipe(Effect.provide(httpLayer)) + return yield* body(events) + }), + ) + +describe("MapleEvents (server)", () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it.effect("posts one NDJSON line per event with the ingest key and identity fields", () => + Effect.gen(function* () { + const { posts, layer } = stubClient() + yield* withEvents(config, layer, (events) => + Effect.gen(function* () { + yield* events.track("plan_started", { + userId: "user_1", + groupId: "org_1", + visitorId: "v-1", + sessionId: "s-1", + timestamp: new Date("2026-08-17T10:00:00.000Z"), + url: "https://app.example.com/billing", + pagePath: "/billing", + attributes: { plan: "startup", seats: 5, trial: false, dropped: undefined }, + }) + yield* events.track(" padded ") + yield* events.flush + }), + ) + + expect(posts).toHaveLength(1) + const [batch] = posts + expect(batch!.url).toBe("https://ingest.test/v1/events") + expect(batch!.headers.authorization).toBe("Bearer secret") + expect(batch!.contentType).toBe("application/x-ndjson") + expect(batch!.lines).toEqual([ + { + name: "plan_started", + timestamp: "2026-08-17T10:00:00.000Z", + source: "server", + service_name: "billing", + visitor_id: "v-1", + user_id: "user_1", + group_id: "org_1", + session_id: "s-1", + url: "https://app.example.com/billing", + page_path: "/billing", + attributes: { plan: "startup", seats: "5", trial: "false" }, + }, + expect.objectContaining({ + name: "padded", + source: "server", + service_name: "billing", + visitor_id: "", + user_id: "", + group_id: "", + session_id: "", + url: "", + page_path: "", + attributes: {}, + }), + ]) + expect(typeof batch!.lines[1]!.timestamp).toBe("string") + }), + ) + + it.effect("batches: nothing is posted until flush, and flush with an empty buffer posts nothing", () => + Effect.gen(function* () { + const { posts, layer } = stubClient() + yield* withEvents(config, layer, (events) => + Effect.gen(function* () { + yield* events.flush + expect(posts).toHaveLength(0) + yield* events.track("a") + yield* events.track("b") + yield* events.track("c") + expect(posts).toHaveLength(0) + yield* events.flush + expect(posts).toHaveLength(1) + expect(posts[0]!.lines.map((line) => line.name)).toEqual(["a", "b", "c"]) + yield* events.flush + expect(posts).toHaveLength(1) + }), + ) + }), + ) + + it.effect("flushes early once maxBatchSize events are buffered", () => + Effect.gen(function* () { + const { posts, layer } = stubClient() + yield* withEvents({ ...config, maxBatchSize: 2 }, layer, (events) => + Effect.gen(function* () { + yield* events.track("a") + yield* events.track("b") + // The size-triggered flush is forked; let it run. + yield* Effect.yieldNow + yield* Effect.yieldNow + expect(posts).toHaveLength(1) + expect(posts[0]!.lines.map((line) => line.name)).toEqual(["a", "b"]) + }), + ) + }), + ) + + it.effect("drains the buffer when the scope closes", () => + Effect.gen(function* () { + const { posts, layer } = stubClient() + const scope = yield* Scope.make() + const events = yield* make(config).pipe(Effect.provide(layer), Scope.provide(scope)) + yield* events.track("shutdown_event") + expect(posts).toHaveLength(0) + yield* Scope.close(scope, undefined as never) + expect(posts).toHaveLength(1) + expect(posts[0]!.lines[0]!.name).toBe("shutdown_event") + }), + ) + + it.effect("never fails the caller: a rejected batch is dropped with a warning", () => + Effect.gen(function* () { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}) + const { posts, layer } = stubClient(401) + yield* withEvents(config, layer, (events) => + Effect.gen(function* () { + yield* events.track("a") + yield* events.flush + yield* events.flush + }), + ) + expect(posts).toHaveLength(1) + expect(warn).toHaveBeenCalledTimes(1) + expect(String(warn.mock.calls[0]![0])).toContain("401") + }), + ) + + it.effect("ignores an empty event name and drops events without an ingest key", () => + Effect.gen(function* () { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}) + const { posts, layer } = stubClient() + yield* withEvents({ ...config, ingestKey: undefined }, layer, (events) => + Effect.gen(function* () { + yield* events.track(" ") + yield* events.track("real") + yield* events.flush + }), + ) + expect(posts).toHaveLength(0) + expect(warn.mock.calls.map((call) => String(call[0]))).toEqual([ + expect.stringContaining("non-empty event name"), + expect.stringContaining("no ingest key"), + ]) + }), + ) + + it.effect("is usable as the MapleEvents service", () => + Effect.gen(function* () { + const { posts, layer } = stubClient() + const serviceLayer = Layer.effect(MapleEvents, make(config)).pipe(Layer.provide(layer)) + yield* MapleEvents.use((events) => + Effect.gen(function* () { + yield* events.track("via_layer", { userId: "u" }) + yield* events.flush + }), + ).pipe(Effect.provide(serviceLayer)) + expect(posts[0]!.lines[0]).toMatchObject({ name: "via_layer", user_id: "u" }) + }), + ) + + it("makeHandle: fire-and-forget track, flush and dispose over fetch", async () => { + const bodies: Array = [] + const original = globalThis.fetch + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push( + typeof init?.body === "string" + ? init.body + : new TextDecoder().decode(init?.body as ArrayBuffer), + ) + return new Response(null, { status: 200 }) + }) as typeof fetch + try { + const handle = makeHandle(config) + handle.track("signup_completed", { userId: "user_9" }) + await handle.flush() + expect(bodies).toHaveLength(1) + expect(JSON.parse(bodies[0]!.trim())).toMatchObject({ + name: "signup_completed", + user_id: "user_9", + }) + handle.track("last_one") + await handle.dispose() + expect(bodies).toHaveLength(2) + expect(JSON.parse(bodies[1]!.trim())).toMatchObject({ name: "last_one" }) + } finally { + globalThis.fetch = original + } + }) +}) diff --git a/packages/effect-sdk/src/server/events.ts b/packages/effect-sdk/src/server/events.ts new file mode 100644 index 000000000..dbc101f2c --- /dev/null +++ b/packages/effect-sdk/src/server/events.ts @@ -0,0 +1,299 @@ +// Server-side product events — the backend twin of the browser SDK's `track()`. +// +// Browser `track()` calls ride the session-events stream. A backend has no +// session, but it is the only place that knows about the events that matter +// most for a funnel — `signup_completed`, `plan_started` — so this posts them +// straight to the ingest gateway's `POST /v1/events` (NDJSON, one event per +// line), keyed to a person by `userId` / `groupId` / `visitorId`. +// +// import { MapleEvents } from "@maple-dev/effect-sdk/server" +// +// // Effect: provide `MapleEvents.layer(...)`, then +// const events = yield* MapleEvents.MapleEvents +// yield* events.track("plan_started", { userId, groupId, attributes: { plan } }) +// +// // Promise-land: +// const events = MapleEvents.makeHandle({ serviceName: "billing" }) +// events.track("plan_started", { userId, attributes: { plan } }) +// await events.dispose() // on shutdown: final flush +// +// Events are buffered and flushed on an interval, on batch size, and on scope +// close / `dispose()`. `track` never fails and never throws into the caller — +// a broken ingest endpoint drops the batch and warns (rate-limited). + +import { coerceTrackProps, MAX_EVENT_NAME_LENGTH, type TrackProps } from "@maple/browser-session/props" +import { + Cause, + Context, + Duration, + Effect, + Layer, + ManagedRuntime, + Option, + Redacted, + Schedule, + Schema, + Semaphore, +} from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { SDK_VERSION } from "../version.js" +import { resolveResource } from "./resource.js" + +export type { TrackProps } + +/** Default flush cadence, matching the OTLP exporters' 5s interval. */ +const DEFAULT_FLUSH_INTERVAL = Duration.seconds(5) +/** Batch size that triggers an early flush. */ +const DEFAULT_MAX_BATCH_SIZE = 100 +/** Hard cap on buffered events; oldest are dropped past it. */ +const MAX_BUFFERED_EVENTS = 1_000 +/** How often a failing endpoint may warn. */ +const WARN_INTERVAL_MS = 30_000 + +export interface MapleEventsConfig { + /** + * Stamped as `service_name` on every event. Falls back to + * `OTEL_SERVICE_NAME`, then `"unknown"`. + */ + readonly serviceName?: string | undefined + /** + * Ingest endpoint URL. Falls back to `MAPLE_ENDPOINT`, then + * `OTEL_EXPORTER_OTLP_ENDPOINT`, then the public Maple ingest. + */ + readonly endpoint?: string | undefined + /** Maple ingest key. Falls back to `MAPLE_INGEST_KEY`. Without one, events are dropped. */ + readonly ingestKey?: string | undefined + /** Path appended to `endpoint`. Default `/v1/events`. */ + readonly eventsPath?: string | undefined + /** Background flush cadence. Default 5 seconds. */ + readonly flushInterval?: Duration.Input | undefined + /** Flush as soon as this many events are buffered. Default 100. */ + readonly maxBatchSize?: number | undefined +} + +/** Who and where a server-side event belongs to. Every field is optional. */ +export interface TrackOptions { + /** The signed-in user (`identify()`'s id on the browser side). */ + readonly userId?: string | undefined + /** Company / team / tenant the user acts within. */ + readonly groupId?: string | undefined + /** Browser visitor id, when the backend has it (e.g. forwarded from a cookie). */ + readonly visitorId?: string | undefined + /** Browser session id, when the event belongs to one. */ + readonly sessionId?: string | undefined + /** When the event happened. Default: now. */ + readonly timestamp?: Date | number | undefined + /** Page the event happened on, when known. */ + readonly url?: string | undefined + readonly pagePath?: string | undefined + /** Free-form properties; coerced to strings and capped like the browser `track()`. */ + readonly attributes?: TrackProps | undefined +} + +/** One NDJSON line for `POST /v1/events`. `org_id` comes from the ingest key. */ +interface ProductEventLine { + readonly name: string + readonly timestamp: string + readonly source: "server" + readonly service_name: string + readonly visitor_id: string + readonly user_id: string + readonly group_id: string + readonly session_id: string + readonly url: string + readonly page_path: string + readonly attributes: Record +} + +export interface MapleEventsApi { + /** Buffer one event. Never fails; an invalid name is dropped with a one-shot warning. */ + readonly track: (name: string, options?: TrackOptions) => Effect.Effect + /** Post everything buffered now. Never fails; a failed POST drops the batch and warns. */ + readonly flush: Effect.Effect +} + +export class MapleEvents extends Context.Service()( + "@maple-dev/effect-sdk/MapleEvents", +) {} + +/** The ingest gateway refused (or never answered) a batch. Logged, never surfaced to `track` callers. */ +export class ProductEventsPostError extends Schema.TaggedError()( + "@maple-dev/effect-sdk/ProductEventsPostError", + { + message: Schema.String, + status: Schema.optionalKey(Schema.Number), + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + +const toTimestamp = (value: Date | number | undefined): string => + (value instanceof Date ? value : new Date(value ?? Date.now())).toISOString() + +/** + * Build the client against whatever `HttpClient` is in context, so tests can + * stub the transport and `layer` can supply `FetchHttpClient`. Scoped: the + * background flush fiber and the final drain are tied to the scope. + */ +export const make = Effect.fn("MapleEvents.make")(function* (config: MapleEventsConfig = {}) { + const resolved = yield* resolveResource({ + serviceName: config.serviceName, + endpoint: config.endpoint, + ingestKey: config.ingestKey, + sdkType: "server", + }) + const url = `${resolved.endpoint.replace(/\/$/, "")}${config.eventsPath ?? "/v1/events"}` + const ingestKey = resolved.ingestKey ? Redacted.value(resolved.ingestKey) : undefined + const serviceName = resolved.resource.serviceName + const maxBatchSize = Math.max(1, config.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE) + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk) + // One POST in flight at a time, so a size-triggered flush and the interval + // never race the same buffer. + const flushing = yield* Semaphore.make(1) + + let buffer: Array = [] + let warnedAboutName = false + let warnedAboutKey = false + let lastPostWarnAt = 0 + + const warnPost = (error: ProductEventsPostError): void => { + const now = Date.now() + if (now - lastPostWarnAt < WARN_INTERVAL_MS) return + lastPostWarnAt = now + console.warn(`[MapleEvents] ${error.message} (dropping batch)`, error.cause ?? "") + } + + const post = (lines: ReadonlyArray) => + HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders({ + Authorization: `Bearer ${ingestKey}`, + "user-agent": `maple-effect-sdk-server/${SDK_VERSION}`, + }), + HttpClientRequest.bodyText( + `${lines.map((line) => JSON.stringify(line)).join("\n")}\n`, + "application/x-ndjson", + ), + client.execute, + Effect.asVoid, + Effect.mapError((error) => + error.response !== undefined + ? new ProductEventsPostError({ + message: `POST ${url} → ${error.response.status}`, + status: error.response.status, + }) + : new ProductEventsPostError({ + message: `POST ${url} failed: ${error.message}`, + cause: error, + }), + ), + ) + + const flush: Effect.Effect = flushing.withPermits(1)( + Effect.suspend(() => { + if (buffer.length === 0) return Effect.void + const batch = buffer + buffer = [] + if (ingestKey === undefined) { + if (!warnedAboutKey) { + warnedAboutKey = true + console.warn( + "[MapleEvents] no ingest key — set MAPLE_INGEST_KEY or pass `ingestKey`; dropping events", + ) + } + return Effect.void + } + return post(batch).pipe( + Effect.catchCause((cause) => + Effect.sync(() => { + warnPost( + Option.getOrElse( + Cause.findErrorOption(cause), + () => + new ProductEventsPostError({ + message: `POST ${url} failed`, + cause: Cause.squash(cause), + }), + ), + ) + }), + ), + ) + }), + ) + + const track = (name: string, options: TrackOptions = {}): Effect.Effect => + Effect.sync(() => { + const trimmed = typeof name === "string" ? name.trim() : "" + if (trimmed.length === 0) { + if (!warnedAboutName) { + warnedAboutName = true + console.warn("[MapleEvents] track() needs a non-empty event name; the call was ignored.") + } + return false + } + buffer.push({ + name: trimmed.slice(0, MAX_EVENT_NAME_LENGTH), + timestamp: toTimestamp(options.timestamp), + source: "server", + service_name: serviceName, + visitor_id: options.visitorId ?? "", + user_id: options.userId ?? "", + group_id: options.groupId ?? "", + session_id: options.sessionId ?? "", + url: options.url ?? "", + page_path: options.pagePath ?? "", + attributes: coerceTrackProps(options.attributes), + }) + if (buffer.length > MAX_BUFFERED_EVENTS) buffer.splice(0, buffer.length - MAX_BUFFERED_EVENTS) + return buffer.length >= maxBatchSize + }).pipe( + // The size-triggered flush is forked so `track` returns immediately; + // the semaphore keeps it from overlapping the interval flush. + Effect.flatMap((full) => (full ? Effect.forkDetach(flush) : Effect.void)), + Effect.asVoid, + ) + + const interval = config.flushInterval ?? DEFAULT_FLUSH_INTERVAL + if (Duration.toMillis(interval) > 0) { + yield* Effect.forkScoped(flush.pipe(Effect.schedule(Schedule.spaced(interval)))) + } + // Scope close (layer teardown / `dispose()`) drains what is left. + yield* Effect.addFinalizer(() => flush) + + return { track, flush } satisfies MapleEventsApi +}) + +/** + * `MapleEvents` service backed by `FetchHttpClient`. Provide it alongside + * `Maple.layer`; the scope that owns it flushes on close. + */ +export const layer = (config: MapleEventsConfig = {}): Layer.Layer => + Layer.effect(MapleEvents, make(config)).pipe(Layer.provide(FetchHttpClient.layer)) + +export interface MapleEventsHandle { + /** Buffer one event. Fire-and-forget; never throws. */ + readonly track: (name: string, options?: TrackOptions) => void + /** Post everything buffered now. Never rejects. */ + readonly flush: () => Promise + /** Final flush, then release the runtime. Never rejects. */ + readonly dispose: () => Promise +} + +/** + * Promise-land handle for apps that are not (yet) Effect end to end. Owns its + * own runtime; call `dispose()` on shutdown so the last batch goes out. + */ +export const makeHandle = (config: MapleEventsConfig = {}): MapleEventsHandle => { + const runtime = ManagedRuntime.make(layer(config)) + const swallow = (promise: Promise): Promise => + promise.then( + () => undefined, + () => undefined, + ) + return { + track: (name, options) => { + void swallow(runtime.runPromise(MapleEvents.use((events) => events.track(name, options)))) + }, + flush: () => swallow(runtime.runPromise(MapleEvents.use((events) => events.flush))), + dispose: () => swallow(runtime.dispose()), + } +} diff --git a/packages/effect-sdk/src/server/index.ts b/packages/effect-sdk/src/server/index.ts index 712a8502e..28eecaf33 100644 --- a/packages/effect-sdk/src/server/index.ts +++ b/packages/effect-sdk/src/server/index.ts @@ -3,3 +3,5 @@ export type { MapleConfig } from "./layer.js" export * as MapleFlush from "./flushable.js" export type { FlushableTelemetry, MapleFlushableConfig } from "./flushable.js" export { derivePlatformAttributes, getAutoPlatformAttributes, type PlatformInputs } from "./platform.js" +export * as MapleEvents from "./events.js" +export type { MapleEventsApi, MapleEventsConfig, MapleEventsHandle, TrackOptions } from "./events.js" From 036360a7a9cea260302f7de8e8c2d49120304b59 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:17:37 +0200 Subject: [PATCH 05/15] feat(api): ProductEventsService + Clerk/Autumn webhooks emit signup/plan events signup_completed from Clerk user.created; plan_started/changed/cancelled from Autumn billing.updated (Svix-verified, WebCrypto, no new dep) plus the inline attach path. Web fires plan_checkout_started before the redirect. --- apps/api/alchemy.run.ts | 7 + apps/api/src/platform/Env.ts | 18 ++ apps/api/src/routes/internal/billing.http.ts | 14 +- .../src/routes/internal/query-engine.http.ts | 35 +-- apps/api/src/routes/webhooks/autumn.http.ts | 82 ++++++ apps/api/src/routes/webhooks/clerk.http.ts | 72 +++++ apps/api/src/routes/webhooks/svix-receiver.ts | 63 +++++ .../src/routes/webhooks/webhooks.http.test.ts | 245 ++++++++++++++++++ apps/api/src/runtime/http-graph.ts | 6 + apps/api/src/runtime/service-graph.ts | 7 + apps/api/src/services/billing/plan-events.ts | 34 +++ .../ProductEventsService.test.ts | 137 ++++++++++ .../product-events/ProductEventsService.ts | 201 ++++++++++++++ .../services/product-events/autumn-events.ts | 106 ++++++++ .../services/product-events/clerk-events.ts | 79 ++++++ .../src/services/product-events/svix.test.ts | 66 +++++ apps/api/src/services/product-events/svix.ts | 142 ++++++++++ ...eb-analytics-parity.clickhouse.e2e.test.ts | 20 +- apps/web/src/hooks/use-billing-actions.ts | 13 +- apps/web/src/lib/analytics.ts | 6 + .../domain/src/tinybird/materializations.ts | 1 - 21 files changed, 1324 insertions(+), 30 deletions(-) create mode 100644 apps/api/src/routes/webhooks/autumn.http.ts create mode 100644 apps/api/src/routes/webhooks/clerk.http.ts create mode 100644 apps/api/src/routes/webhooks/svix-receiver.ts create mode 100644 apps/api/src/routes/webhooks/webhooks.http.test.ts create mode 100644 apps/api/src/services/billing/plan-events.ts create mode 100644 apps/api/src/services/product-events/ProductEventsService.test.ts create mode 100644 apps/api/src/services/product-events/ProductEventsService.ts create mode 100644 apps/api/src/services/product-events/autumn-events.ts create mode 100644 apps/api/src/services/product-events/clerk-events.ts create mode 100644 apps/api/src/services/product-events/svix.test.ts create mode 100644 apps/api/src/services/product-events/svix.ts diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 8b369b10e..45fa6cb70 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -294,6 +294,13 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => ...optionalSecret("CLERK_SECRET_KEY"), ...optionalPlain("CLERK_PUBLISHABLE_KEY"), ...optionalSecret("CLERK_JWT_KEY"), + // Svix signing secrets for the public webhook receivers (`/webhooks/clerk`, + // `/webhooks/autumn`); each route answers 503 until its secret is set. + ...optionalSecret("CLERK_WEBHOOK_SECRET"), + ...optionalSecret("AUTUMN_WEBHOOK_SECRET"), + // Server-side product events default to MAPLE_INGEST_KEY (below); set this + // only if the funnel should land in a different org than the API's traces. + ...optionalSecret("MAPLE_PRODUCT_EVENTS_INGEST_KEY"), ...optionalSecret("AUTUMN_SECRET_KEY"), ...optionalSecret("SD_INTERNAL_TOKEN"), ...optionalSecret("INTERNAL_SERVICE_TOKEN"), diff --git a/apps/api/src/platform/Env.ts b/apps/api/src/platform/Env.ts index fc4740cb3..3d839c95f 100644 --- a/apps/api/src/platform/Env.ts +++ b/apps/api/src/platform/Env.ts @@ -42,9 +42,22 @@ export interface EnvConfig { readonly CLERK_SECRET_KEY: Option.Option> readonly CLERK_PUBLISHABLE_KEY: Option.Option readonly CLERK_JWT_KEY: Option.Option> + /** Svix signing secret (`whsec_…`) for `POST /webhooks/clerk`; the route answers 503 while unset. */ + readonly CLERK_WEBHOOK_SECRET: Option.Option> readonly MAPLE_ORG_ID_OVERRIDE: Option.Option readonly AUTUMN_SECRET_KEY: Option.Option> readonly AUTUMN_API_URL: string + /** Svix signing secret (`whsec_…`) for `POST /webhooks/autumn`; the route answers 503 while unset. */ + readonly AUTUMN_WEBHOOK_SECRET: Option.Option> + /** + * Self-observability ingest key (`@maple-dev/effect-sdk` reads the same variable + * for OTLP export). Also the default credential for server-side product events. + */ + readonly MAPLE_INGEST_KEY: Option.Option> + /** Ingest gateway base URL for the SDK's OTLP export; product events reuse it, falling back to MAPLE_INGEST_PUBLIC_URL. */ + readonly MAPLE_ENDPOINT: Option.Option + /** Overrides MAPLE_INGEST_KEY for product events when the dogfood org differs from the tracing org. */ + readonly MAPLE_PRODUCT_EVENTS_INGEST_KEY: Option.Option> readonly SD_INTERNAL_TOKEN: Option.Option> readonly INTERNAL_SERVICE_TOKEN: Option.Option> readonly EMAIL_FROM: string @@ -129,9 +142,14 @@ const envConfig = Config.all({ CLERK_SECRET_KEY: optionalRedacted("CLERK_SECRET_KEY"), CLERK_PUBLISHABLE_KEY: optionalString("CLERK_PUBLISHABLE_KEY"), CLERK_JWT_KEY: optionalRedacted("CLERK_JWT_KEY"), + CLERK_WEBHOOK_SECRET: optionalRedacted("CLERK_WEBHOOK_SECRET"), MAPLE_ORG_ID_OVERRIDE: optionalString("MAPLE_ORG_ID_OVERRIDE"), AUTUMN_SECRET_KEY: optionalRedacted("AUTUMN_SECRET_KEY"), AUTUMN_API_URL: stringWithDefault("AUTUMN_API_URL", "https://api.useautumn.com"), + AUTUMN_WEBHOOK_SECRET: optionalRedacted("AUTUMN_WEBHOOK_SECRET"), + MAPLE_INGEST_KEY: optionalRedacted("MAPLE_INGEST_KEY"), + MAPLE_ENDPOINT: optionalString("MAPLE_ENDPOINT"), + MAPLE_PRODUCT_EVENTS_INGEST_KEY: optionalRedacted("MAPLE_PRODUCT_EVENTS_INGEST_KEY"), SD_INTERNAL_TOKEN: optionalRedacted("SD_INTERNAL_TOKEN"), INTERNAL_SERVICE_TOKEN: optionalRedacted("INTERNAL_SERVICE_TOKEN"), EMAIL_FROM: stringWithDefault("EMAIL_FROM", "Maple "), diff --git a/apps/api/src/routes/internal/billing.http.ts b/apps/api/src/routes/internal/billing.http.ts index d2e41787d..4c66f942f 100644 --- a/apps/api/src/routes/internal/billing.http.ts +++ b/apps/api/src/routes/internal/billing.http.ts @@ -21,6 +21,8 @@ import { readCustomerCached, } from "@/services/billing/autumn-client" import { AutumnClient, type AutumnResult } from "@/services/billing/autumn-http" +import { emitPlanStartedFromAttach } from "@/services/billing/plan-events" +import { ProductEventsService } from "@/services/product-events/ProductEventsService" import { requireAdmin } from "@/services/auth/auth" import { DailySpendService } from "@/services/billing/DailySpendService" @@ -68,6 +70,7 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleInternalApi, "billing", const edgeCache = yield* EdgeCacheService const dailySpend = yield* DailySpendService const autumn = yield* AutumnClient + const productEvents = yield* ProductEventsService // Invalidate on any 2xx, matching `ensureOk` — otherwise a 201/204 from // attach/openCustomerPortal would decode as success yet leave the stale @@ -178,7 +181,16 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleInternalApi, "billing", const result = yield* autumn.attach(tenant.orgId, { planId: payload.planId }) const response = yield* ensureOk(result) yield* invalidateCustomer(tenant.orgId, result) - return yield* decodeUpstream(AttachResult, response) + const attached = yield* decodeUpstream(AttachResult, response) + // Inline (no-redirect) plan start; the Autumn webhook covers the + // Stripe-checkout path. Never fails the request. + yield* emitPlanStartedFromAttach(productEvents, { + orgId: tenant.orgId, + userId: tenant.userId, + planId: payload.planId, + result: attached, + }) + return attached }), ) .handle("previewAttach", ({ payload }) => diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 207386531..9f34cadcd 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -130,13 +130,18 @@ const isMissingServiceOperationsRollup = (error: unknown): boolean => { } /** - * `web_events` is a read-path rollup on a `requiredForIngest: false` migration, - * so a BYO cluster can be perfectly healthy and still not have it — the org just - * hasn't re-applied schema yet. Same shape as the service-operations detector - * above, and the same reason: the fallback has to be automatic and per-org, - * because there is no global moment when every cluster has migrated. + * `product_events` arrives with migration 0016, so a BYO cluster that predates + * it can be perfectly healthy for everything else and still not have the table — + * the org just hasn't re-applied schema yet. Same shape as the + * service-operations detector above, and the same reason: the fallback has to + * be automatic and per-org, because there is no global moment when every cluster + * has migrated. + * + * Only the page-view queries degrade this way — raw `session_events` holds the + * same browser rows. Funnels have no raw counterpart (server and mobile rows + * exist only in `product_events`) and must surface the missing-table error. */ -const isMissingWebEvents = (error: unknown): boolean => { +const isMissingProductEvents = (error: unknown): boolean => { if (typeof error !== "object" || error === null) return false const candidate = error as { readonly _tag?: unknown @@ -146,7 +151,7 @@ const isMissingWebEvents = (error: unknown): boolean => { return ( candidate._tag === "@maple/http/errors/WarehouseConfigError" && (candidate.clickhouseType === "UNKNOWN_TABLE" || - (typeof candidate.message === "string" && /web_events/i.test(candidate.message))) + (typeof candidate.message === "string" && /product_events/i.test(candidate.message))) ) } @@ -193,9 +198,9 @@ const makeRollupFallback = }), ) -const withWebEventsFallback = makeRollupFallback( - isMissingWebEvents, - "web_events is absent on this cluster; reading raw session_events. Apply ClickHouse schema to restore the fast path.", +const withProductEventsFallback = makeRollupFallback( + isMissingProductEvents, + "product_events is absent on this cluster; reading raw session_events. Apply ClickHouse schema to restore the fast path.", ) const withServiceOperationsFallback = makeRollupFallback( @@ -1602,7 +1607,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("webAnalyticsSummary", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const row = yield* withWebEventsFallback( + const row = yield* withProductEventsFallback( (t, pl) => runQueryFirst(Queries.webAnalyticsSummary, t, pl), (t, pl) => runQueryFirst(Queries.webAnalyticsSummaryRaw, t, pl), tenant, @@ -1625,7 +1630,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("webAnalyticsTimeseries", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* withWebEventsFallback( + const rows = yield* withProductEventsFallback( (t, pl) => runQuery(Queries.webAnalyticsTimeseries, t, pl), (t, pl) => runQuery(Queries.webAnalyticsTimeseriesRaw, t, pl), tenant, @@ -1647,7 +1652,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("webAnalyticsPageviews", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* withWebEventsFallback( + const rows = yield* withProductEventsFallback( (t, pl) => runQuery(Queries.webAnalyticsPageviews, t, pl), (t, pl) => runQuery(Queries.webAnalyticsPageviewsRaw, t, pl), tenant, @@ -1665,7 +1670,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("webAnalyticsPages", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* withWebEventsFallback( + const rows = yield* withProductEventsFallback( (t, pl) => runQuery(Queries.webAnalyticsPages, t, pl), (t, pl) => runQuery(Queries.webAnalyticsPagesRaw, t, pl), tenant, @@ -1684,7 +1689,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("webAnalyticsBreakdowns", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* withWebEventsFallback( + const rows = yield* withProductEventsFallback( (t, pl) => runQuery(Queries.webAnalyticsBreakdowns, t, pl), (t, pl) => runQuery(Queries.webAnalyticsBreakdownsRaw, t, pl), tenant, diff --git a/apps/api/src/routes/webhooks/autumn.http.ts b/apps/api/src/routes/webhooks/autumn.http.ts new file mode 100644 index 000000000..db279a444 --- /dev/null +++ b/apps/api/src/routes/webhooks/autumn.http.ts @@ -0,0 +1,82 @@ +import { Effect, Option } from "effect" +import { HttpRouter, type HttpServerRequest } from "effect/unstable/http" +import { Env } from "@/platform/Env" +import { + AUTUMN_BILLING_UPDATED, + decodeAutumnBillingUpdated, + decodeAutumnEnvelope, + planEventsFromBillingUpdated, +} from "@/services/product-events/autumn-events" +import { ProductEventsService } from "@/services/product-events/ProductEventsService" +import { receiveSvixWebhook, webhookText } from "./svix-receiver" + +/** + * Autumn webhook receiver: `billing.updated` → `plan_started` / `plan_changed` + * / `plan_cancelled` product events, `group_id` = the Autumn customer id, which + * is the Maple org id. Public route; authenticity is the Svix signature + * (`AUTUMN_WEBHOOK_SECRET`). Other event types are acknowledged with 200. + */ +const ROUTE = "/webhooks/autumn" + +export const AutumnWebhookRouter = HttpRouter.use((router) => + Effect.gen(function* () { + const env = yield* Env + const productEvents = yield* ProductEventsService + + const handle = Effect.fn("AutumnWebhook.receive")(function* ( + req: HttpServerRequest.HttpServerRequest, + ) { + yield* Effect.annotateCurrentSpan({ "http.request.method": req.method, "http.route": ROUTE }) + + const received = yield* receiveSvixWebhook({ + provider: "autumn", + secret: env.AUTUMN_WEBHOOK_SECRET, + request: req, + }) + if (received._tag === "rejected") return received.response + + const envelope = yield* decodeAutumnEnvelope(received.body).pipe(Effect.option) + if (Option.isNone(envelope)) { + yield* Effect.annotateCurrentSpan({ + "http.response.status_code": 400, + "maple.webhook.outcome": "rejected", + "maple.webhook.reason": "parse_rejected", + }) + return webhookText("Unrecognized payload", 400) + } + yield* Effect.annotateCurrentSpan({ "maple.webhook.event": envelope.value.type }) + + if (envelope.value.type === AUTUMN_BILLING_UPDATED) { + const data = yield* decodeAutumnBillingUpdated(envelope.value.data).pipe( + Effect.tapError((error) => + Effect.logInfo("Autumn billing.updated payload failed to decode").pipe( + Effect.annotateLogs({ error: String(error) }), + ), + ), + Effect.option, + ) + if (Option.isSome(data)) { + const events = planEventsFromBillingUpdated(data.value, { + id: envelope.value.id ?? received.messageId, + occurred_at: envelope.value.occurred_at, + }) + yield* Effect.annotateCurrentSpan({ + orgId: data.value.customer_id, + "maple.webhook.outcome": "handled", + "maple.webhook.emitted": events.length, + }) + yield* Effect.forEach(events, (event) => productEvents.track(event), { discard: true }) + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "parse_rejected" }) + } + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "ignored" }) + } + + yield* Effect.annotateCurrentSpan({ "http.response.status_code": 200 }) + return webhookText("ok", 200) + }) + + yield* router.add("POST", ROUTE, handle) + }), +) diff --git a/apps/api/src/routes/webhooks/clerk.http.ts b/apps/api/src/routes/webhooks/clerk.http.ts new file mode 100644 index 000000000..e84f5543d --- /dev/null +++ b/apps/api/src/routes/webhooks/clerk.http.ts @@ -0,0 +1,72 @@ +import { Effect, Option } from "effect" +import { HttpRouter, type HttpServerRequest } from "effect/unstable/http" +import { Env } from "@/platform/Env" +import { + decodeClerkEnvelope, + decodeClerkUserCreated, + signupCompletedEvent, +} from "@/services/product-events/clerk-events" +import { ProductEventsService } from "@/services/product-events/ProductEventsService" +import { receiveSvixWebhook, webhookText } from "./svix-receiver" + +/** + * Clerk webhook receiver: `user.created` → `signup_completed` product event. + * Public route; authenticity is the Svix signature (`CLERK_WEBHOOK_SECRET`). + * Any other event type is acknowledged with 200 so Clerk does not retry it. + */ +const ROUTE = "/webhooks/clerk" + +export const ClerkWebhookRouter = HttpRouter.use((router) => + Effect.gen(function* () { + const env = yield* Env + const productEvents = yield* ProductEventsService + + const handle = Effect.fn("ClerkWebhook.receive")(function* ( + req: HttpServerRequest.HttpServerRequest, + ) { + yield* Effect.annotateCurrentSpan({ "http.request.method": req.method, "http.route": ROUTE }) + + const received = yield* receiveSvixWebhook({ + provider: "clerk", + secret: env.CLERK_WEBHOOK_SECRET, + request: req, + }) + if (received._tag === "rejected") return received.response + + const envelope = yield* decodeClerkEnvelope(received.body).pipe(Effect.option) + if (Option.isNone(envelope)) { + yield* Effect.annotateCurrentSpan({ + "http.response.status_code": 400, + "maple.webhook.outcome": "rejected", + "maple.webhook.reason": "parse_rejected", + }) + return webhookText("Unrecognized payload", 400) + } + yield* Effect.annotateCurrentSpan({ "maple.webhook.event": envelope.value.type }) + + if (envelope.value.type === "user.created") { + const user = yield* decodeClerkUserCreated(envelope.value.data).pipe( + Effect.tapError((error) => + Effect.logInfo("Clerk user.created payload failed to decode").pipe( + Effect.annotateLogs({ error: String(error) }), + ), + ), + Effect.option, + ) + if (Option.isSome(user)) { + yield* productEvents.track(signupCompletedEvent(user.value, envelope.value.timestamp)) + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "handled" }) + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "parse_rejected" }) + } + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "ignored" }) + } + + yield* Effect.annotateCurrentSpan({ "http.response.status_code": 200 }) + return webhookText("ok", 200) + }) + + yield* router.add("POST", ROUTE, handle) + }), +) diff --git a/apps/api/src/routes/webhooks/svix-receiver.ts b/apps/api/src/routes/webhooks/svix-receiver.ts new file mode 100644 index 000000000..d2218d36b --- /dev/null +++ b/apps/api/src/routes/webhooks/svix-receiver.ts @@ -0,0 +1,63 @@ +import { Clock, Effect, Option, Redacted, Result } from "effect" +import { HttpServerResponse, type HttpServerRequest } from "effect/unstable/http" +import { readSvixHeaders, verifySvixSignature } from "@/services/product-events/svix" + +/** + * Shared receive step for Svix-delivered webhooks (Clerk, Autumn). NOT behind + * auth — authenticity is the Svix signature over the raw body. Every outcome is + * an HTTP response: 503 while the secret is unset (so the provider retries once + * it is configured), 401 on a bad/stale signature, 400 on an empty body, and + * `{ _tag: "verified" }` with the raw body for the caller to decode. Span + * attributes carry the outcome so a misconfigured secret is visible in traces. + */ + +export type SvixReceiveOutcome = + | { readonly _tag: "verified"; readonly body: string; readonly messageId: string } + | { readonly _tag: "rejected"; readonly response: HttpServerResponse.HttpServerResponse } + +const textResponse = (body: string, status: number) => HttpServerResponse.text(body, { status }) + +export const receiveSvixWebhook = (options: { + readonly provider: "clerk" | "autumn" + readonly secret: Option.Option> + readonly request: HttpServerRequest.HttpServerRequest +}): Effect.Effect => + Effect.gen(function* () { + const attr = (key: string) => `maple.webhook.${key}` + yield* Effect.annotateCurrentSpan({ [attr("provider")]: options.provider }) + + const reject = (status: number, reason: string, body: string) => + Effect.annotateCurrentSpan({ + "http.response.status_code": status, + [attr("outcome")]: "rejected", + [attr("reason")]: reason, + }).pipe(Effect.as({ _tag: "rejected", response: textResponse(body, status) })) + + if (Option.isNone(options.secret)) { + return yield* reject(503, "secret_unset", "Webhook receiver is not configured") + } + + const bodyOpt = yield* options.request.text.pipe(Effect.option) + if (Option.isNone(bodyOpt) || bodyOpt.value.length === 0) { + return yield* reject(400, "empty_body", "Missing request body") + } + + const headers = readSvixHeaders(options.request.headers) + const nowMs = yield* Clock.currentTimeMillis + const verified = yield* Effect.result( + verifySvixSignature({ + secret: Redacted.value(options.secret.value), + headers, + body: bodyOpt.value, + nowMs, + }), + ) + if (Result.isFailure(verified)) { + return yield* reject(401, verified.failure.reason, "Invalid signature") + } + + yield* Effect.annotateCurrentSpan({ [attr("message_id")]: headers.id ?? "" }) + return { _tag: "verified", body: bodyOpt.value, messageId: headers.id ?? "" } + }) + +export const webhookText = textResponse diff --git a/apps/api/src/routes/webhooks/webhooks.http.test.ts b/apps/api/src/routes/webhooks/webhooks.http.test.ts new file mode 100644 index 000000000..f130cfb53 --- /dev/null +++ b/apps/api/src/routes/webhooks/webhooks.http.test.ts @@ -0,0 +1,245 @@ +import { assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Context, Effect, Layer } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { Env } from "@/platform/Env" +import { ProductEventsService, type ProductEventInput } from "@/services/product-events/ProductEventsService" +import { signSvix } from "@/services/product-events/svix" +import { AutumnWebhookRouter } from "./autumn.http" +import { ClerkWebhookRouter } from "./clerk.http" + +const CLERK_SECRET = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" +const AUTUMN_SECRET = "whsec_" + Buffer.alloc(24, 7).toString("base64") + +const makeConfig = (extra: Record) => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 5).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + ...extra, + }), + ) + +const recordingProductEvents = () => { + const tracked: Array = [] + const layer = Layer.succeed(ProductEventsService, { + enabled: true, + track: (event) => Effect.sync(() => void tracked.push(event)), + }) + return { tracked, layer } +} + +const makeRouterLayer = ( + router: typeof ClerkWebhookRouter, + config: Record, + productEvents: Layer.Layer, +) => router.pipe(Layer.provide(productEvents), Layer.provide(Env.layer), Layer.provide(makeConfig(config))) + +const signedHeaders = (secret: string, body: string, nowMs: number, id = "msg_test") => + Effect.gen(function* () { + const timestamp = String(Math.floor(nowMs / 1000)) + const signature = yield* signSvix(secret, id, timestamp, body) + return { "svix-id": id, "svix-timestamp": timestamp, "svix-signature": `v1,${signature}` } + }) + +const post = ( + handler: (request: Request, context: Context.Context) => Promise, + path: string, + body: string, + headers: Record, +) => + Effect.promise(() => + handler( + new Request(`http://api.localhost${path}`, { method: "POST", body, headers }), + Context.empty(), + ), + ) + +const CLERK_USER_CREATED = JSON.stringify({ + type: "user.created", + timestamp: 1_700_000_000_000, + data: { + id: "user_2abc", + created_at: 1_700_000_000_000, + primary_email_address_id: "idn_2", + email_addresses: [ + { id: "idn_1", email_address: "personal@gmail.com" }, + { id: "idn_2", email_address: "Dev@Example.COM" }, + ], + external_accounts: [{ provider: "oauth_github" }], + }, +}) + +const AUTUMN_BILLING_UPDATED = JSON.stringify({ + type: "billing.updated", + data: { + object: "billing.updated", + customer_id: "org_42", + plan_changes: [ + { + action: "activated", + subscription: { + plan_id: "startup", + status: "active", + past_due: false, + started_at: 1_761_840_000_000, + canceled_at: null, + expires_at: null, + trial_ends_at: null, + current_period_start: 1_761_840_000_000, + current_period_end: 1_764_432_000_000, + }, + previous_attributes: null, + item_changes: [], + }, + { + action: "expired", + subscription: { + plan_id: "free", + status: "expired", + past_due: false, + started_at: 1_759_248_000_000, + canceled_at: 1_761_840_000_000, + expires_at: 1_761_840_000_000, + trial_ends_at: null, + current_period_start: null, + current_period_end: null, + }, + previous_attributes: { status: "active" }, + item_changes: [], + }, + ], + tags: [], + }, +}) + +describe("ClerkWebhookRouter", () => { + it.effect( + "503s while unconfigured, 401s a bad signature, and emits signup_completed for user.created", + () => + Effect.gen(function* () { + const events = recordingProductEvents() + const unconfigured = HttpRouter.toWebHandler( + makeRouterLayer(ClerkWebhookRouter, {}, events.layer), + { + disableLogger: true, + }, + ) + yield* Effect.gen(function* () { + const response = yield* post( + unconfigured.handler, + "/webhooks/clerk", + CLERK_USER_CREATED, + {}, + ) + assert.strictEqual(response.status, 503) + }).pipe(Effect.ensuring(Effect.promise(unconfigured.dispose))) + + const configured = HttpRouter.toWebHandler( + makeRouterLayer(ClerkWebhookRouter, { CLERK_WEBHOOK_SECRET: CLERK_SECRET }, events.layer), + { disableLogger: true }, + ) + yield* Effect.gen(function* () { + const now = Date.now() + const missing = yield* post(configured.handler, "/webhooks/clerk", CLERK_USER_CREATED, {}) + assert.strictEqual(missing.status, 401) + + const stale = yield* signedHeaders(CLERK_SECRET, CLERK_USER_CREATED, now - 10 * 60 * 1000) + const staleResponse = yield* post( + configured.handler, + "/webhooks/clerk", + CLERK_USER_CREATED, + stale, + ) + assert.strictEqual(staleResponse.status, 401) + + const tampered = yield* signedHeaders(CLERK_SECRET, CLERK_USER_CREATED, now) + const tamperedResponse = yield* post( + configured.handler, + "/webhooks/clerk", + CLERK_USER_CREATED.replace("user_2abc", "user_evil"), + tampered, + ) + assert.strictEqual(tamperedResponse.status, 401) + assert.strictEqual(events.tracked.length, 0) + + const ok = yield* signedHeaders(CLERK_SECRET, CLERK_USER_CREATED, now) + const accepted = yield* post( + configured.handler, + "/webhooks/clerk", + CLERK_USER_CREATED, + ok, + ) + assert.strictEqual(accepted.status, 200) + assert.deepStrictEqual(events.tracked, [ + { + name: "signup_completed", + userId: "user_2abc", + timestamp: 1_700_000_000_000, + attributes: { sign_up_source: "github", email_domain: "example.com" }, + }, + ]) + + // Other event types are acknowledged and ignored. + const other = JSON.stringify({ type: "session.created", data: { id: "sess_1" } }) + const otherHeaders = yield* signedHeaders(CLERK_SECRET, other, now, "msg_other") + const ignored = yield* post(configured.handler, "/webhooks/clerk", other, otherHeaders) + assert.strictEqual(ignored.status, 200) + assert.strictEqual(events.tracked.length, 1) + }).pipe(Effect.ensuring(Effect.promise(configured.dispose))) + }), + ) +}) + +describe("AutumnWebhookRouter", () => { + it.effect("emits plan_started for an activated plan and plan_cancelled for the expired one", () => + Effect.gen(function* () { + const events = recordingProductEvents() + const { handler, dispose } = HttpRouter.toWebHandler( + makeRouterLayer(AutumnWebhookRouter, { AUTUMN_WEBHOOK_SECRET: AUTUMN_SECRET }, events.layer), + { disableLogger: true }, + ) + yield* Effect.gen(function* () { + const now = Date.now() + const wrongSecret = yield* signedHeaders(CLERK_SECRET, AUTUMN_BILLING_UPDATED, now) + const rejected = yield* post(handler, "/webhooks/autumn", AUTUMN_BILLING_UPDATED, wrongSecret) + assert.strictEqual(rejected.status, 401) + + const ok = yield* signedHeaders(AUTUMN_SECRET, AUTUMN_BILLING_UPDATED, now, "msg_autumn_1") + const accepted = yield* post(handler, "/webhooks/autumn", AUTUMN_BILLING_UPDATED, ok) + assert.strictEqual(accepted.status, 200) + assert.deepStrictEqual(events.tracked, [ + { + name: "plan_started", + groupId: "org_42", + timestamp: 1_761_840_000_000, + attributes: { + plan_id: "startup", + trigger: "webhook", + kind: "subscription", + subscription_started_at: "1761840000000", + webhook_message_id: "msg_autumn_1", + }, + }, + { + name: "plan_cancelled", + groupId: "org_42", + timestamp: undefined, + attributes: { + plan_id: "free", + trigger: "webhook", + kind: "subscription", + subscription_started_at: "1759248000000", + webhook_message_id: "msg_autumn_1", + }, + }, + ]) + }).pipe(Effect.ensuring(Effect.promise(dispose))) + }), + ) +}) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index bce96b60e..8db710e21 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -29,6 +29,8 @@ import { ScraperInternalRouter } from "@/routes/v1/scraper-internal.http" import { HttpSessionReplaysLive } from "@/routes/v1/session-replay.http" import { SlackCallbackRouter, SlackInternalRouter } from "@/routes/v1/slack-integration.http" import { VcsWebhookRouter } from "@/routes/v1/vcs-webhook.http" +import { AutumnWebhookRouter } from "@/routes/webhooks/autumn.http" +import { ClerkWebhookRouter } from "@/routes/webhooks/clerk.http" import { HttpV2AlertDeliveriesLive } from "@/routes/v2/alert-deliveries.http" import { HttpV2AlertDestinationsLive } from "@/routes/v2/alert-destinations.http" import { HttpV2AlertIncidentsLive } from "@/routes/v2/alert-incidents.http" @@ -148,6 +150,8 @@ export const AllRoutes = Layer.mergeAll( PrometheusScrapeProxyRouter, ScraperInternalRouter, VcsWebhookRouter, + ClerkWebhookRouter, + AutumnWebhookRouter, McpLive, HealthRouter, DocsRoute, @@ -203,5 +207,7 @@ export const ApiObservabilityLive = Layer.mergeAll( "x-api-key", "x-hub-signature", "x-hub-signature-256", + // Svix (Clerk / Autumn webhooks): replayable alongside its body within the tolerance window. + "svix-signature", ]), ) diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index d0c234ab6..14e0a935a 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -58,6 +58,7 @@ import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" import { OrgMembersService } from "@/services/org/OrgMembersService" import { OrganizationService } from "@/services/org/OrganizationService" import { SetupAuditService } from "@/services/org/SetupAuditService" +import { ProductEventsService } from "@/services/product-events/ProductEventsService" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" @@ -249,9 +250,15 @@ const DailySpendServiceLive = DailySpendService.layer.pipe(Layer.provideMerge(Wa // configured"), so an unconfigured local worker still boots. const AutumnClientLive = AutumnClient.layer.pipe(Layer.provide(InfraLive)) +// Server-side product events (signup/plan funnel) — the Clerk/Autumn webhook +// receivers and the billing `attach` route emit through it. Builds without an +// ingest key (every `track` is then a logged no-op). +const ProductEventsServiceLive = ProductEventsService.layer.pipe(Layer.provide(InfraLive)) + const MainServicesLive = Layer.mergeAll( CoreServicesLive, AutumnClientLive, + ProductEventsServiceLive, DailySpendServiceLive, CloudflareAnalyticsServiceLive, WarehouseQueryServiceLive, diff --git a/apps/api/src/services/billing/plan-events.ts b/apps/api/src/services/billing/plan-events.ts new file mode 100644 index 000000000..5f1608e0b --- /dev/null +++ b/apps/api/src/services/billing/plan-events.ts @@ -0,0 +1,34 @@ +import { Effect } from "effect" +import type { AttachResult } from "@maple/domain/http" +import type { ProductEventsApi } from "@/services/product-events/ProductEventsService" + +/** + * `plan_started` for the no-redirect `attach` outcome. + * + * `billing.attach` with `redirect_mode: "if_required"` either returns a Stripe + * `paymentUrl` (the plan starts later, on the Stripe → Autumn side, and the + * Autumn `billing.updated` webhook is the truth) or applies the change inline + * for a customer with a payment method on file. Both paths eventually produce + * a webhook, so this emit is the LOW-LATENCY complement, not the only signal: + * it carries `trigger=attach` and the acting `user_id`, which the webhook can + * never know. Consumers dedupe on `(group_id, plan_id)` within a window — see + * `autumn-events.ts` for why there is no `subscription_id` to key on. + */ +export const emitPlanStartedFromAttach = ( + productEvents: ProductEventsApi, + input: { + readonly orgId: string + readonly userId: string + readonly planId: string + readonly result: AttachResult + }, +): Effect.Effect => { + const redirected = input.result.paymentUrl !== undefined && input.result.paymentUrl !== null + if (redirected) return Effect.void + return productEvents.track({ + name: "plan_started", + userId: input.userId, + groupId: input.orgId, + attributes: { plan_id: input.planId, trigger: "attach" }, + }) +} diff --git a/apps/api/src/services/product-events/ProductEventsService.test.ts b/apps/api/src/services/product-events/ProductEventsService.test.ts new file mode 100644 index 000000000..e8f1b4414 --- /dev/null +++ b/apps/api/src/services/product-events/ProductEventsService.test.ts @@ -0,0 +1,137 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { HttpClient, HttpClientResponse, type HttpClientRequest } from "effect/unstable/http" +import { makeProductEvents, toProductEventLine } from "./ProductEventsService" + +interface Captured { + readonly url: string + readonly method: string + readonly headers: Record + readonly body: string +} + +/** An HttpClient that records the request and answers with the given statuses in order. */ +const stubClient = (statuses: ReadonlyArray) => { + const captured: Array = [] + let call = 0 + const client = HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + const body = request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "" + captured.push({ url: request.url, method: request.method, headers: { ...request.headers }, body }) + const status = statuses[Math.min(call, statuses.length - 1)] ?? 200 + call += 1 + return HttpClientResponse.fromWeb(request, new Response(null, { status })) + }), + ) + return { client, captured } +} + +const NOW = Date.UTC(2026, 7, 17, 12, 0, 0) + +describe("ProductEventsService", () => { + it("projects an event onto the /v1/events wire line", () => { + const line = toProductEventLine( + { + name: "plan_started", + userId: "user_1", + groupId: "org_1", + attributes: { plan_id: "startup", "": "dropped", empty: "" }, + }, + NOW, + ) + assert.deepStrictEqual(line, { + name: "plan_started", + timestamp: "2026-08-17T12:00:00.000Z", + source: "server", + service_name: "maple-api", + user_id: "user_1", + group_id: "org_1", + attributes: { plan_id: "startup" }, + }) + }) + + it("caps attributes at 32 keys and 1024-char values, and omits empty ids", () => { + const attributes: Record = {} + for (let i = 0; i < 40; i++) attributes[`k${i}`] = "x".repeat(2000) + const line = toProductEventLine({ name: "signup_completed", userId: "", attributes }, NOW) + assert.isUndefined(line.user_id) + assert.isUndefined(line.group_id) + assert.strictEqual(Object.keys(line.attributes ?? {}).length, 32) + assert.strictEqual(line.attributes?.k0?.length, 1024) + }) + + it.effect("POSTs one NDJSON line with the bearer ingest key", () => + Effect.gen(function* () { + const { client, captured } = stubClient([200]) + const events = makeProductEvents({ + httpClient: client, + endpoint: "https://ingest.example.test/", + ingestKey: "maple_sk_test", + }) + assert.isTrue(events.enabled) + yield* events.track({ + name: "signup_completed", + userId: "user_abc", + attributes: { email_domain: "example.com" }, + timestamp: NOW, + }) + assert.strictEqual(captured.length, 1) + const request = captured[0]! + assert.strictEqual(request.method, "POST") + assert.strictEqual(request.url, "https://ingest.example.test/v1/events") + assert.strictEqual(request.headers.authorization, "Bearer maple_sk_test") + assert.match(request.headers["content-type"] ?? "", /application\/x-ndjson/) + assert.isTrue(request.body.endsWith("\n")) + const lines = request.body.trimEnd().split("\n") + assert.strictEqual(lines.length, 1) + assert.deepStrictEqual(JSON.parse(lines[0]!), { + name: "signup_completed", + timestamp: "2026-08-17T12:00:00.000Z", + source: "server", + service_name: "maple-api", + user_id: "user_abc", + attributes: { email_domain: "example.com" }, + }) + }), + ) + + it.effect("retries once on 5xx and never fails the caller", () => + Effect.gen(function* () { + const { client, captured } = stubClient([503, 503, 503]) + const events = makeProductEvents({ + httpClient: client, + endpoint: "https://ingest.example.test", + ingestKey: "k", + }) + yield* events.track({ name: "plan_started", groupId: "org_1" }) + assert.strictEqual(captured.length, 2) + }), + ) + + it.effect("does not retry a 4xx", () => + Effect.gen(function* () { + const { client, captured } = stubClient([400]) + const events = makeProductEvents({ + httpClient: client, + endpoint: "https://ingest.example.test", + ingestKey: "k", + }) + yield* events.track({ name: "plan_started", groupId: "org_1" }) + assert.strictEqual(captured.length, 1) + }), + ) + + it.effect("is a no-op without an ingest key", () => + Effect.gen(function* () { + const { client, captured } = stubClient([200]) + const events = makeProductEvents({ + httpClient: client, + endpoint: "https://ingest.example.test", + ingestKey: undefined, + }) + assert.isFalse(events.enabled) + yield* events.track({ name: "plan_started", groupId: "org_1" }) + assert.strictEqual(captured.length, 0) + }), + ) +}) diff --git a/apps/api/src/services/product-events/ProductEventsService.ts b/apps/api/src/services/product-events/ProductEventsService.ts new file mode 100644 index 000000000..e8fca80b6 --- /dev/null +++ b/apps/api/src/services/product-events/ProductEventsService.ts @@ -0,0 +1,201 @@ +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { Env } from "@/platform/Env" + +/** + * Server-side product events for Maple's own funnel (`signup_completed`, + * `plan_started`, …). Each call POSTs one NDJSON line to the ingest gateway's + * `POST /v1/events`, authenticated with the same ingest key the API uses for its + * own telemetry, so the rows land in the dogfood org next to the browser + * `track()` events from `maple-web` / `maple-landing`. + * + * `track` never fails the caller: the request is bounded (timeout + one retry) + * and any failure is logged at debug and swallowed. It runs INLINE rather than + * on a forked fiber — on Workers a fiber forked into the request scope is + * interrupted the moment the response is returned, and the worker never hands + * `ExecutionContext.waitUntil` into the Effect graph (see + * `OrgClickHouseSettingsService.refreshCachedSettings`), so "fire-and-forget" + * would silently drop most events. Callers that cannot afford ~1 RTT wrap it in + * `forkRequestScoped` themselves. + * + * Wire contract (mirrors `/v1/sessionEvents` sanitising): `name` ≤ 128 chars, + * not `$`-prefixed; ≤ 32 attributes, key ≤ 64, value ≤ 1024. Excess is trimmed + * here so a malformed emit degrades to a shorter row rather than a 400. + */ + +export class ProductEventsError extends Schema.TaggedError()( + "@maple/api/services/product-events/ProductEventsError", + { + message: Schema.String, + status: Schema.optionalKey(Schema.Number), + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + +export type ProductEventName = + | "signup_completed" + | "plan_checkout_started" + | "plan_started" + | "plan_changed" + | "plan_cancelled" + +export interface ProductEventInput { + readonly name: ProductEventName + readonly userId?: string | undefined + readonly groupId?: string | undefined + readonly attributes?: Readonly> | undefined + /** Epoch ms; defaults to now. Webhooks pass the provider's own timestamp. */ + readonly timestamp?: number | undefined +} + +/** One NDJSON line as the ingest gateway expects it. Exported for tests. */ +export interface ProductEventLine { + readonly name: string + readonly timestamp: string + readonly source: "server" + readonly service_name: "maple-api" + readonly user_id?: string + readonly group_id?: string + readonly attributes?: Record +} + +export interface ProductEventsApi { + /** Emit an event. Never fails; a dropped event is logged at debug. */ + readonly track: (event: ProductEventInput) => Effect.Effect + /** Whether an ingest key is configured — `track` is a no-op otherwise. */ + readonly enabled: boolean +} + +export const PRODUCT_EVENTS_PATH = "/v1/events" +const MAX_NAME_LENGTH = 128 +const MAX_ATTRIBUTES = 32 +const MAX_ATTRIBUTE_KEY = 64 +const MAX_ATTRIBUTE_VALUE = 1024 +const REQUEST_TIMEOUT = "2500 millis" +const RETRIES = 1 + +const trimTrailingSlash = (url: string) => url.replace(/\/+$/, "") + +const sanitizeAttributes = ( + attributes: Readonly> | undefined, +): Record | undefined => { + if (attributes === undefined) return undefined + const out: Record = {} + let count = 0 + for (const [key, value] of Object.entries(attributes)) { + if (count >= MAX_ATTRIBUTES) break + if (key.length === 0 || key.length > MAX_ATTRIBUTE_KEY) continue + if (typeof value !== "string" || value.length === 0) continue + out[key] = value.length > MAX_ATTRIBUTE_VALUE ? value.slice(0, MAX_ATTRIBUTE_VALUE) : value + count += 1 + } + return count === 0 ? undefined : out +} + +/** Pure projection input → wire line. Exported so the shape is testable without HTTP. */ +export const toProductEventLine = (event: ProductEventInput, nowMs: number): ProductEventLine => { + const attributes = sanitizeAttributes(event.attributes) + return { + name: event.name.slice(0, MAX_NAME_LENGTH), + timestamp: new Date(event.timestamp ?? nowMs).toISOString(), + source: "server", + service_name: "maple-api", + ...(event.userId !== undefined && event.userId.length > 0 ? { user_id: event.userId } : undefined), + ...(event.groupId !== undefined && event.groupId.length > 0 + ? { group_id: event.groupId } + : undefined), + ...(attributes !== undefined ? { attributes } : undefined), + } +} + +const toError = (message: string) => (cause: unknown) => + new ProductEventsError({ + message: cause instanceof Error ? `${message}: ${cause.message}` : message, + cause, + }) + +const isRetryable = (error: ProductEventsError) => + // Network failures, timeouts and 5xx/429 retry once; a 4xx is our bug and + // retrying it is noise. + error.status === undefined || error.status >= 500 || error.status === 429 + +export const makeProductEvents = (options: { + readonly httpClient: HttpClient.HttpClient + readonly endpoint: string + readonly ingestKey: string | undefined +}): ProductEventsApi => { + const url = `${trimTrailingSlash(options.endpoint)}${PRODUCT_EVENTS_PATH}` + const ingestKey = options.ingestKey + + const post = (line: ProductEventLine): Effect.Effect => + Effect.gen(function* () { + const request = HttpClientRequest.post(url, { + headers: { Authorization: `Bearer ${ingestKey}` }, + }).pipe(HttpClientRequest.bodyText(`${JSON.stringify(line)}\n`, "application/x-ndjson")) + const response = yield* options.httpClient + .execute(request) + .pipe(Effect.mapError(toError("Product event request failed"))) + yield* Effect.annotateCurrentSpan({ "http.response.status_code": response.status }) + if (response.status < 200 || response.status >= 300) { + return yield* new ProductEventsError({ + message: `Ingest gateway answered ${response.status} for ${line.name}`, + status: response.status, + }) + } + }).pipe( + Effect.timeoutOrElse({ + duration: REQUEST_TIMEOUT, + orElse: () => + new ProductEventsError({ message: `Product event request timed out (${line.name})` }), + }), + ) + + const track = Effect.fn("ProductEvents.track")(function* (event: ProductEventInput) { + yield* Effect.annotateCurrentSpan({ + "maple.product_event.name": event.name, + "maple.product_event.enabled": ingestKey !== undefined, + }) + if (ingestKey === undefined) { + yield* Effect.logDebug("Product event dropped: no ingest key configured").pipe( + Effect.annotateLogs({ event: event.name }), + ) + return + } + const now = yield* Clock.currentTimeMillis + const line = toProductEventLine(event, now) + yield* post(line).pipe( + Effect.retry({ times: RETRIES, while: isRetryable }), + Effect.catchCause((cause) => + Effect.logDebug("Product event dropped").pipe( + Effect.annotateLogs({ event: event.name, cause: String(cause) }), + Effect.tap(() => Effect.annotateCurrentSpan({ "maple.product_event.dropped": true })), + ), + ), + ) + }) + + return { track, enabled: ingestKey !== undefined } +} + +export class ProductEventsService extends Context.Service()( + "@maple/api/services/product-events/ProductEventsService", + { + make: Effect.gen(function* () { + const env = yield* Env + const httpClient = yield* HttpClient.HttpClient + const ingestKey = Option.getOrUndefined( + Option.orElse(env.MAPLE_PRODUCT_EVENTS_INGEST_KEY, () => env.MAPLE_INGEST_KEY), + ) + const endpoint = Option.getOrElse(env.MAPLE_ENDPOINT, () => env.MAPLE_INGEST_PUBLIC_URL) + return makeProductEvents({ + httpClient, + endpoint, + ingestKey: ingestKey === undefined ? undefined : Redacted.value(ingestKey), + }) + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) + /** Drop everything — for tests and non-HTTP entrypoints that never emit. */ + static readonly noop = Layer.succeed(this, { track: () => Effect.void, enabled: false }) +} diff --git a/apps/api/src/services/product-events/autumn-events.ts b/apps/api/src/services/product-events/autumn-events.ts new file mode 100644 index 000000000..d411582f7 --- /dev/null +++ b/apps/api/src/services/product-events/autumn-events.ts @@ -0,0 +1,106 @@ +import { Schema } from "effect" +import type { ProductEventInput, ProductEventName } from "./ProductEventsService" + +/** + * Autumn webhook payload → product events. Autumn delivers through Svix with a + * `{ type, data }` envelope; `billing.updated` fires "when a customer's plans + * change — activated, scheduled, updated, or expired" and carries one + * `plan_changes[]` entry per affected plan (docs.useautumn.com/documentation/webhooks). + * + * Mapping (Autumn `customer_id` IS the Maple org id — see `autumn-client.ts`): + * - `activated` → `plan_started` (a `scheduled` plan is not started yet) + * - `updated` → `plan_changed` + * - `expired` → `plan_cancelled` + * + * Autumn's payload has NO subscription id — only `plan_id` and lifecycle + * timestamps — so `subscription_id` is not attributable here. `started_at` + * (epoch ms) is carried as `subscription_started_at`; together with `plan_id` + * it identifies one subscription across the webhook and the inline `attach` + * emit, which is what a consumer would dedupe on. + */ + +const AutumnSubscription = Schema.Struct({ + plan_id: Schema.String, + status: Schema.optionalKey(Schema.String), + started_at: Schema.optionalKey(Schema.NullOr(Schema.Number)), + trial_ends_at: Schema.optionalKey(Schema.NullOr(Schema.Number)), + canceled_at: Schema.optionalKey(Schema.NullOr(Schema.Number)), +}) + +const AutumnPurchase = Schema.Struct({ + plan_id: Schema.String, + status: Schema.optionalKey(Schema.String), +}) + +const AutumnPlanChange = Schema.Struct({ + action: Schema.String, + subscription: Schema.optionalKey(Schema.NullOr(AutumnSubscription)), + purchase: Schema.optionalKey(Schema.NullOr(AutumnPurchase)), +}) + +export const AutumnBillingUpdatedData = Schema.Struct({ + customer_id: Schema.String, + entity_id: Schema.optionalKey(Schema.NullOr(Schema.String)), + plan_changes: Schema.Array(AutumnPlanChange), + tags: Schema.optionalKey(Schema.Array(Schema.String)), +}) + +export const AutumnWebhookEnvelope = Schema.Struct({ + type: Schema.String, + id: Schema.optionalKey(Schema.String), + /** Epoch ms; present on some event types. */ + occurred_at: Schema.optionalKey(Schema.Number), + data: Schema.Unknown, +}) +export type AutumnWebhookEnvelope = Schema.Schema.Type + +export const decodeAutumnEnvelope = Schema.decodeUnknownEffect(Schema.fromJsonString(AutumnWebhookEnvelope)) +export const decodeAutumnBillingUpdated = Schema.decodeUnknownEffect(AutumnBillingUpdatedData) +type AutumnBillingUpdatedData = Schema.Schema.Type + +export const AUTUMN_BILLING_UPDATED = "billing.updated" + +const ACTION_EVENT: ReadonlyMap = new Map([ + ["activated", "plan_started"], + ["updated", "plan_changed"], + ["expired", "plan_cancelled"], +]) + +/** Ready-to-track events for one `billing.updated` delivery — possibly none. */ +export const planEventsFromBillingUpdated = ( + data: AutumnBillingUpdatedData, + envelope: { readonly id?: string | undefined; readonly occurred_at?: number | undefined }, +): ReadonlyArray => { + // Entity-scoped plans (per-seat / per-project sub-customers) are not org plans. + if (data.entity_id !== undefined && data.entity_id !== null && data.entity_id.length > 0) return [] + const events: Array = [] + for (const change of data.plan_changes) { + const name = ACTION_EVENT.get(change.action) + if (name === undefined) continue + const planId = change.subscription?.plan_id ?? change.purchase?.plan_id + if (planId === undefined) continue + const startedAt = change.subscription?.started_at ?? undefined + events.push({ + name, + groupId: data.customer_id, + timestamp: name === "plan_started" ? (startedAt ?? envelope.occurred_at) : envelope.occurred_at, + attributes: { + plan_id: planId, + trigger: "webhook", + kind: + change.subscription !== undefined && change.subscription !== null + ? "subscription" + : "purchase", + ...(startedAt !== undefined && startedAt !== null + ? { subscription_started_at: String(startedAt) } + : undefined), + ...(change.subscription?.trial_ends_at != null ? { trial: "true" } : undefined), + ...(envelope.id !== undefined ? { webhook_message_id: envelope.id } : undefined), + ...(data.tags !== undefined && data.tags.length > 0 + ? { tags: data.tags.join(",") } + : undefined), + }, + }) + } + return events +} diff --git a/apps/api/src/services/product-events/clerk-events.ts b/apps/api/src/services/product-events/clerk-events.ts new file mode 100644 index 000000000..c193820ef --- /dev/null +++ b/apps/api/src/services/product-events/clerk-events.ts @@ -0,0 +1,79 @@ +import { Schema } from "effect" +import type { ProductEventInput } from "./ProductEventsService" + +/** + * Clerk webhook payload → product event. Only `user.created` is mapped today + * (`signup_completed`); everything else is acknowledged and ignored. The + * schema is deliberately loose — Clerk adds fields freely and a decode failure + * here would 400 a delivery we would otherwise have handled. + * + * No raw email leaves this module: `email_domain` is the only address-derived + * attribute, and it is enough to separate consumer from company signups. + */ + +const ClerkEmailAddress = Schema.Struct({ + id: Schema.optionalKey(Schema.String), + email_address: Schema.optionalKey(Schema.String), +}) + +const ClerkExternalAccount = Schema.Struct({ + provider: Schema.optionalKey(Schema.String), +}) + +export const ClerkUserCreatedData = Schema.Struct({ + id: Schema.String, + email_addresses: Schema.optionalKey(Schema.Array(ClerkEmailAddress)), + primary_email_address_id: Schema.optionalKey(Schema.NullOr(Schema.String)), + external_accounts: Schema.optionalKey(Schema.Array(ClerkExternalAccount)), + /** Epoch ms. */ + created_at: Schema.optionalKey(Schema.Number), +}) + +export const ClerkWebhookEnvelope = Schema.Struct({ + type: Schema.String, + data: Schema.Unknown, + /** Epoch ms of the event (Clerk stamps this on every delivery). */ + timestamp: Schema.optionalKey(Schema.Number), +}) +export type ClerkWebhookEnvelope = Schema.Schema.Type + +export const decodeClerkEnvelope = Schema.decodeUnknownEffect(Schema.fromJsonString(ClerkWebhookEnvelope)) +export const decodeClerkUserCreated = Schema.decodeUnknownEffect(ClerkUserCreatedData) +type ClerkUserCreatedData = Schema.Schema.Type + +const emailDomain = (data: ClerkUserCreatedData): string | undefined => { + const addresses = data.email_addresses ?? [] + const primary = + addresses.find((entry) => entry.id !== undefined && entry.id === data.primary_email_address_id) ?? + addresses[0] + const at = primary?.email_address?.lastIndexOf("@") ?? -1 + if (primary?.email_address === undefined || at < 0) return undefined + const domain = primary.email_address + .slice(at + 1) + .trim() + .toLowerCase() + return domain.length > 0 ? domain : undefined +} + +/** `oauth_google` → `google`; no external account → `email`. */ +const signUpSource = (data: ClerkUserCreatedData): string => { + const provider = data.external_accounts?.find((entry) => entry.provider !== undefined)?.provider + if (provider === undefined || provider.length === 0) return "email" + return provider.startsWith("oauth_") ? provider.slice("oauth_".length) : provider +} + +export const signupCompletedEvent = ( + data: ClerkUserCreatedData, + envelopeTimestamp: number | undefined, +): ProductEventInput => { + const domain = emailDomain(data) + return { + name: "signup_completed", + userId: data.id, + timestamp: data.created_at ?? envelopeTimestamp, + attributes: { + sign_up_source: signUpSource(data), + ...(domain !== undefined ? { email_domain: domain } : undefined), + }, + } +} diff --git a/apps/api/src/services/product-events/svix.test.ts b/apps/api/src/services/product-events/svix.test.ts new file mode 100644 index 000000000..d8c5a4db9 --- /dev/null +++ b/apps/api/src/services/product-events/svix.test.ts @@ -0,0 +1,66 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { readSvixHeaders, signSvix, verifySvixSignature } from "./svix" + +// The vector the svix client libraries ship in their own test suites. +const SECRET = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" +const MSG_ID = "msg_p5jXN8AQM9LWM0D4loKWxJek" +const TIMESTAMP = "1614265330" +const BODY = '{"test": 2432232314}' +const SIGNATURE = "v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=" +const NOW_MS = 1614265330 * 1000 + +const headers = (overrides: Partial> = {}) => + readSvixHeaders({ + "svix-id": MSG_ID, + "svix-timestamp": TIMESTAMP, + "svix-signature": SIGNATURE, + ...overrides, + }) + +const verify = (options: Partial[0]> = {}) => + verifySvixSignature({ secret: SECRET, headers: headers(), body: BODY, nowMs: NOW_MS, ...options }) + +describe("svix", () => { + it.effect("signs the known vector", () => + Effect.gen(function* () { + const signature = yield* signSvix(SECRET, MSG_ID, TIMESTAMP, BODY) + assert.strictEqual(`v1,${signature}`, SIGNATURE) + }), + ) + + it.effect("accepts a valid delivery", () => + Effect.gen(function* () { + yield* verify() + }), + ) + + it.effect("accepts when the matching signature is one of several (secret rotation)", () => + Effect.gen(function* () { + yield* verify({ headers: headers({ "svix-signature": `v1,AAAA= ${SIGNATURE} v2,ignored` }) }) + }), + ) + + it.effect("rejects a tampered body", () => + Effect.gen(function* () { + const error = yield* Effect.flip(verify({ body: '{"test": 1}' })) + assert.strictEqual(error.reason, "signature_mismatch") + }), + ) + + it.effect("rejects a stale timestamp (> 5 min) even with a valid signature", () => + Effect.gen(function* () { + const error = yield* Effect.flip(verify({ nowMs: NOW_MS + 6 * 60 * 1000 })) + assert.strictEqual(error.reason, "stale_timestamp") + }), + ) + + it.effect("rejects missing headers and a wrong secret", () => + Effect.gen(function* () { + const missing = yield* Effect.flip(verify({ headers: headers({ "svix-signature": "" }) })) + assert.strictEqual(missing.reason, "missing_headers") + const wrongSecret = yield* Effect.flip(verify({ secret: "whsec_AAAAAAAAAAAAAAAAAAAAAA==" })) + assert.strictEqual(wrongSecret.reason, "signature_mismatch") + }), + ) +}) diff --git a/apps/api/src/services/product-events/svix.ts b/apps/api/src/services/product-events/svix.ts new file mode 100644 index 000000000..999b81a05 --- /dev/null +++ b/apps/api/src/services/product-events/svix.ts @@ -0,0 +1,142 @@ +import { Effect, Schema } from "effect" + +/** + * Svix (Standard Webhooks) signature verification, shared by the Clerk and + * Autumn receivers — both deliver through Svix. + * + * Scheme: `svix-signature` carries space-separated `v1,` entries (one + * per active secret during rotation). Each is HMAC-SHA256 over + * `${svix-id}.${svix-timestamp}.${rawBody}` keyed with the base64-decoded + * secret after the `whsec_` prefix. `svix-timestamp` is Unix seconds and is + * rejected outside a ±5 minute window so a captured delivery cannot be replayed. + * + * WebCrypto only — no `svix` dependency, and `crypto.subtle` is present on + * Workers, Node ≥ 19 and vitest alike. + */ + +export const SVIX_TOLERANCE_SECONDS = 5 * 60 + +export type SvixRejection = + | "missing_headers" + | "bad_timestamp" + | "stale_timestamp" + | "bad_secret" + | "signature_mismatch" + +export class SvixVerificationError extends Schema.TaggedError()( + "@maple/api/services/product-events/SvixVerificationError", + { + message: Schema.String, + reason: Schema.Literals([ + "missing_headers", + "bad_timestamp", + "stale_timestamp", + "bad_secret", + "signature_mismatch", + ]), + }, +) {} + +export interface SvixHeaders { + readonly id: string | undefined + readonly timestamp: string | undefined + readonly signature: string | undefined +} + +export const readSvixHeaders = (headers: Readonly>): SvixHeaders => ({ + id: headers["svix-id"], + timestamp: headers["svix-timestamp"], + signature: headers["svix-signature"], +}) + +const decodeBase64 = (value: string): Uint8Array | undefined => { + try { + const binary = atob(value) + const bytes = new Uint8Array(new ArrayBuffer(binary.length)) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return bytes + } catch { + return undefined + } +} + +const encodeBase64 = (bytes: ArrayBuffer): string => { + let binary = "" + for (const byte of new Uint8Array(bytes)) binary += String.fromCharCode(byte) + return btoa(binary) +} + +const constantTimeEqual = (a: string, b: string): boolean => { + if (a.length !== b.length) return false + let diff = 0 + for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i) + return diff === 0 +} + +const secretBytes = (secret: string): Uint8Array | undefined => { + const raw = secret.startsWith("whsec_") ? secret.slice("whsec_".length) : secret + const bytes = decodeBase64(raw.trim()) + return bytes === undefined || bytes.length === 0 ? undefined : bytes +} + +/** Exported for tests and for anyone needing to sign a payload the Svix way. */ +export const signSvix = (secret: string, id: string, timestamp: string, body: string) => + Effect.gen(function* () { + const keyBytes = secretBytes(secret) + if (keyBytes === undefined) { + return yield* new SvixVerificationError({ + message: "Webhook secret is not valid base64", + reason: "bad_secret", + }) + } + const key = yield* Effect.promise(() => + crypto.subtle.importKey("raw", keyBytes, { name: "HMAC", hash: "SHA-256" }, false, ["sign"]), + ) + const mac = yield* Effect.promise(() => + crypto.subtle.sign("HMAC", key, new TextEncoder().encode(`${id}.${timestamp}.${body}`)), + ) + return encodeBase64(mac) + }) + +export const verifySvixSignature = (options: { + readonly secret: string + readonly headers: SvixHeaders + readonly body: string + readonly nowMs: number + readonly toleranceSeconds?: number | undefined +}): Effect.Effect => + Effect.gen(function* () { + const { id, timestamp, signature } = options.headers + if (!id || !timestamp || !signature) { + return yield* new SvixVerificationError({ + message: "Missing svix headers", + reason: "missing_headers", + }) + } + const ts = Number(timestamp) + if (!Number.isInteger(ts) || ts <= 0) { + return yield* new SvixVerificationError({ + message: "Malformed svix-timestamp", + reason: "bad_timestamp", + }) + } + const tolerance = options.toleranceSeconds ?? SVIX_TOLERANCE_SECONDS + if (Math.abs(Math.floor(options.nowMs / 1000) - ts) > tolerance) { + return yield* new SvixVerificationError({ + message: "svix-timestamp outside tolerance", + reason: "stale_timestamp", + }) + } + const expected = yield* signSvix(options.secret, id, timestamp, options.body) + const provided = signature + .split(" ") + .map((entry) => entry.trim()) + .filter((entry) => entry.startsWith("v1,")) + .map((entry) => entry.slice(3)) + if (!provided.some((candidate) => constantTimeEqual(candidate, expected))) { + return yield* new SvixVerificationError({ + message: "Signature mismatch", + reason: "signature_mismatch", + }) + } + }) diff --git a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts index 9cb556bed..95b002e8f 100644 --- a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. // Raw-vs-rollup parity for web analytics. // -// `web_events` is a performance change, not a semantics change: every one of the +// `product_events` is a performance change, not a semantics change: every one of the // five web-analytics queries must return byte-identical numbers whether it reads // raw `session_events` or the rollup. The analyzer sweep next door proves the // rollup SQL *parses*; only running both against the same rows proves they @@ -19,7 +19,7 @@ // - The navigation semi-join changing which sessions it selects, which is the // path the breakdown fan-out inlines twelve times. // - The MV's WHERE dropping or admitting the wrong event types: clicks and -// network rows must never reach `web_events`, and custom rows must. +// network rows must never reach `product_events`, and custom rows must. import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" import * as CH from "@maple/query-engine/ch" @@ -37,7 +37,7 @@ const ORG_ID = "org_web_analytics_parity" /** * The seed window is anchored to *now*, not to a fixed calendar date. * - * `session_events` and `web_events` both carry a 30-day TTL, and ClickHouse + * `session_events` and `product_events` both carry a 30-day TTL, and ClickHouse * enforces it at insert time — a hardcoded date silently drops every seeded row * the moment it ages past the horizon, both tables end up empty, and every * comparison below passes by comparing nothing to nothing. This test was written @@ -266,7 +266,7 @@ const assertSourcesSwapped = (rawSql: string, rollupSql: string): void => { return } assert.notInclude(rollupSql, "FROM session_events", "the rollup variant still reads the raw table") - assert.include(rollupSql, "FROM web_events", "the rollup variant does not read web_events") + assert.include(rollupSql, "FROM product_events", "the rollup variant does not read product_events") } /** Order-insensitive comparison keyed on the row's dimension columns — the @@ -369,8 +369,8 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () ) }) - it("populates web_events from the materialized view with only navigation and custom rows", async () => { - const rows = await runJson("SELECT Kind, count() AS n FROM web_events GROUP BY Kind ORDER BY Kind") + it("populates product_events from the materialized view with only navigation and custom rows", async () => { + const rows = await runJson("SELECT Kind, count() AS n FROM product_events GROUP BY Kind ORDER BY Kind") const byKind = Object.fromEntries(rows.map((row) => [String(row.Kind), Number(row.n)])) const expectedNavigation = SEED_EVENTS.filter((row) => row.type === "navigation").length const expectedCustom = SEED_EVENTS.filter((row) => row.type === "custom").length @@ -378,7 +378,7 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () // The reserved-name collision lands as a custom row, not a page view. const collision = await runJson( - "SELECT Kind FROM web_events WHERE EventName = '$pageview' AND Kind = 'custom'", + "SELECT Kind FROM product_events WHERE EventName = '$pageview' AND Kind = 'custom'", ) assert.lengthOf(collision, 1, "track('$pageview') must stay a custom event") }) @@ -386,7 +386,7 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () it("pre-extracts Host and PagePath exactly as the read-time functions would", async () => { const drift = await runJson( `SELECT count() AS n - FROM web_events + FROM product_events WHERE Host != domain(Url) OR PagePath != path(Url)`, ) assert.strictEqual(Number(drift[0]?.n), 0, "write-time URL parsing diverged from read-time") @@ -396,8 +396,8 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () for (const query of QUERIES) { for (const filterCase of FILTER_CASES) { it(`${query.name} agrees across sources — ${filterCase.label}`, async () => { - const rawSql = query.compile({ ...filterCase.filters, useWebEvents: false }) - const rollupSql = query.compile({ ...filterCase.filters, useWebEvents: true }) + const rawSql = query.compile({ ...filterCase.filters, useProductEvents: false }) + const rollupSql = query.compile({ ...filterCase.filters, useProductEvents: true }) assertSourcesSwapped(rawSql, rollupSql) const [rawRows, rollupRows] = await Promise.all([runJson(rawSql), runJson(rollupSql)]) diff --git a/apps/web/src/hooks/use-billing-actions.ts b/apps/web/src/hooks/use-billing-actions.ts index 885617722..604862f54 100644 --- a/apps/web/src/hooks/use-billing-actions.ts +++ b/apps/web/src/hooks/use-billing-actions.ts @@ -1,6 +1,7 @@ import { useCallback } from "react" import { Cause, Exit } from "effect" import { toastManager } from "@maple/ui/components/ui/toast" +import { trackProduct } from "@/lib/analytics" import { useAtomSet } from "@/lib/effect-atom" import { AttachRequest, @@ -41,13 +42,19 @@ export function useBillingActions() { const portalSet = useAtomSet(openCustomerPortalMutation, { mode: "promiseExit" }) const attach = useCallback( - async ({ planId }: { planId: string }): Promise => - unwrap( + async ({ planId }: { planId: string }): Promise => { + const result = unwrap( await attachSet({ payload: new AttachRequest({ planId }), reactivityKeys: MUTATION_KEYS, }), - ), + ) + // Every caller redirects to `paymentUrl` when present, so this is the last + // moment the session is still ours: record checkout intent before the tab + // leaves for Stripe. `plan_started` itself is emitted server-side. + if (result.paymentUrl) trackProduct("plan_checkout_started", { plan_id: planId }) + return result + }, [attachSet], ) diff --git a/apps/web/src/lib/analytics.ts b/apps/web/src/lib/analytics.ts index 24e5b1a4c..f74df5d9a 100644 --- a/apps/web/src/lib/analytics.ts +++ b/apps/web/src/lib/analytics.ts @@ -22,6 +22,12 @@ export type ProductEvent = | "api_key_created" | "dashboard_created" | "chat_message_sent" + /** + * Client-side intent signal fired right before the Stripe redirect. The + * server-side `plan_started` (Autumn webhook / inline attach) is the truth for + * "started a plan"; this only exists to size checkout drop-off. + */ + | "plan_checkout_started" /** * Record a product event. Never throws and never awaits — the SDK buffers and diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 1d24c5a39..d1c4badfe 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -27,7 +27,6 @@ import { serviceOperationsHourly, productEvents, identityLinks, - sessionReplays, } from "./datasources" import { DB_NAMESPACE_ATTR_SQL, From 381f2eb3fd1f0b9bd9f0d8939b9faef85f97d93c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:17:45 +0200 Subject: [PATCH 06/15] test(domain): drop invalid requiredForIngest assertion on 0016 --- packages/domain/src/clickhouse/migrations/index.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index a3d77bce6..a73db7e01 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -44,7 +44,6 @@ describe("ClickHouse migrations", () => { // columns and `product_events` directly, so a BYO-CH org must apply it // before ingest routes there again. expect(clickHouseSchemaVersion).toBe("16") - expect(migration_0016_product_events.requiredForIngest).toBeUndefined() expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) expect(migration_0015_service_overview_minutely.requiredForIngest).toBe(false) From fcfc7002a1679a3deb9140c9315cbaca37d1dc73 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 12:27:49 +0200 Subject: [PATCH 07/15] feat(query-engine): funnels over product_events; readers flip off web_events windowFunnel/sequenceMatch in clickhouse-builder; productEventsFunnelQuery, productEventsFunnelBreakdownQuery, productEventNamesQuery with a person key stitched through identity_links. Page-view SQL byte-identical apart from the table. Catalog baseline regenerated; parity e2e extended with funnel cases. --- .env.example | 7 + ...eb-analytics-parity.clickhouse.e2e.test.ts | 234 ++++++- lib/clickhouse-builder/docs/expressions.md | 13 + .../src/ch/core-dsl.test.ts | 40 ++ .../src/ch/functions/aggregate.ts | 53 ++ .../src/ch/functions/index.ts | 3 + lib/clickhouse-builder/src/ch/index.ts | 3 + .../src/__sql_baseline__/catalog.sql | 425 +++++++++++- .../query-engine/src/ch/builder-fixtures.ts | 153 ++++- packages/query-engine/src/ch/index.ts | 24 + .../src/ch/queries/product-events.test.ts | 261 +++++++ .../src/ch/queries/product-events.ts | 643 ++++++++++++++++++ .../src/ch/queries/web-analytics.ts | 46 +- packages/query-engine/src/registry/queries.ts | 24 +- packages/query-engine/src/sql-catalog.test.ts | 2 + packages/query-engine/src/sql-catalog.ts | 4 +- 16 files changed, 1853 insertions(+), 82 deletions(-) create mode 100644 packages/query-engine/src/ch/queries/product-events.test.ts create mode 100644 packages/query-engine/src/ch/queries/product-events.ts diff --git a/.env.example b/.env.example index 85314a293..4823745b6 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,9 @@ MAPLE_ROOT_PASSWORD=change-me # CLERK_PUBLISHABLE_KEY=pk_test_xxx # Optional: networkless JWT verification # CLERK_JWT_KEY=-----BEGIN PUBLIC KEY-----... +# Svix signing secret of the Clerk webhook endpoint pointing at POST /webhooks/clerk +# (user.created → signup_completed product event). Route answers 503 while unset. +# CLERK_WEBHOOK_SECRET=whsec_xxx # Required when MAPLE_AUTH_MODE=self_hosted MAPLE_DEFAULT_ORG_ID=default @@ -104,6 +107,9 @@ INGEST_REQUIRE_TLS=false # Billing (Autumn) # AUTUMN_SECRET_KEY=am_sk_test_xxx +# Svix signing secret of the Autumn webhook endpoint pointing at POST /webhooks/autumn +# (billing.updated → plan_started / plan_changed / plan_cancelled). 503 while unset. +# AUTUMN_WEBHOOK_SECRET=whsec_xxx # A configured Autumn account is authoritative for ingest entitlements. Checks # fail open when Autumn is unavailable. @@ -180,6 +186,7 @@ INGEST_REQUIRE_TLS=false # MAPLE_ENVIRONMENT=local # "local" = no export; any other value enables OTLP # MAPLE_ENDPOINT=http://127.0.0.1:3474 # Ingest gateway endpoint (enriches with org_id) # MAPLE_INGEST_KEY= # Ingest key (maple_pk_* or maple_sk_*) for self-observability +# MAPLE_PRODUCT_EVENTS_INGEST_KEY= # Optional override for server-side product events (defaults to MAPLE_INGEST_KEY) # COMMIT_SHA= # Git commit SHA for service version # Scraper internal token diff --git a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts index 95b002e8f..1457d989c 100644 --- a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts @@ -37,7 +37,7 @@ const ORG_ID = "org_web_analytics_parity" /** * The seed window is anchored to *now*, not to a fixed calendar date. * - * `session_events` and `product_events` both carry a 30-day TTL, and ClickHouse + * `session_events` carries a 30-day TTL (`product_events` a longer one), and ClickHouse * enforces it at insert time — a hardcoded date silently drops every seeded row * the moment it ages past the horizon, both tables end up empty, and every * comparison below passes by comparing nothing to nothing. This test was written @@ -333,7 +333,11 @@ const QUERIES: ReadonlyArray<{ }, ] -describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () => { +// One database for both suites in this file — migrations are the slow part. +// File-level so the parity suite's teardown cannot run before the funnel suite +// below has started; guarded because file-level hooks run even when every +// `describe` is skipped. +if (clickhouseE2eEnabled) { beforeAll(async () => { await clickhouseExec(`CREATE DATABASE ${database}`) await applyRealMigrations(database) @@ -343,7 +347,9 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () afterAll(async () => { await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) }, 30_000) +} +describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () => { it("seeds land in both tables", async () => { // The guard against a vacuous suite. Every comparison below is an equality // between two result sets, so two empty result sets pass every one of them — @@ -419,3 +425,227 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () } } }) + +// Funnels over product_events. +// +// Not a raw-vs-rollup parity — funnels have no raw counterpart, since server +// rows exist only in `product_events`. What is under test is the arithmetic: +// per-step counts computed by hand from a small seed versus what the compiled +// SQL returns, across every person-key resolution, the session-step branch, the +// window bound, and both breakdown sources. +// +// The seed is its own org so the web-analytics counts above are untouched: +// f1 v1 (anonymous) referred by t.co / twitter / → /pricing → signup_started +// f2 v2 referred by t.co / → /pricing +// f3 v3 referred by google.com /pricing +// f5 v1 + u1 direct (identified later) /dashboard ← links v1 → u1 +// server rows: plan_started for u1 (plan=startup) and for u9 (unlinked) + +const FUNNEL_ORG_ID = "org_funnel_parity" +const funnelWindow = { orgId: FUNNEL_ORG_ID, startTime: START_TIME, endTime: END_TIME } + +interface FunnelSeedEvent extends SeedEvent { + readonly visitorId: string + readonly userId: string +} + +const fev = ( + sessionId: string, + visitorId: string, + userId: string, + ts: string, + seq: number, + type: string, + url: string, + message = "", +): FunnelSeedEvent => ({ sessionId, visitorId, userId, ts, seq, type, url, message }) + +const FUNNEL_EVENTS: ReadonlyArray = [ + fev("f1", "v1", "", at(HOUR_MS), 0, "navigation", "https://maple.dev/"), + fev("f1", "v1", "", at(HOUR_MS + MINUTE_MS), 1, "navigation", "https://maple.dev/pricing"), + fev("f1", "v1", "", at(HOUR_MS + 2 * MINUTE_MS), 2, "custom", "https://maple.dev/pricing", "signup_started"), + fev("f2", "v2", "", at(2 * HOUR_MS), 0, "navigation", "https://maple.dev/"), + fev("f2", "v2", "", at(2 * HOUR_MS + MINUTE_MS), 1, "navigation", "https://maple.dev/pricing"), + fev("f3", "v3", "", at(3 * HOUR_MS), 0, "navigation", "https://maple.dev/pricing"), + fev("f5", "v1", "u1", at(5 * HOUR_MS), 0, "navigation", "https://app.maple.dev/dashboard"), +] + +const FUNNEL_SESSIONS: ReadonlyArray = [ + sess("f1", 1, { + visitorId: "v1", + referrerHost: "t.co", + utmSource: "twitter", + country: "DE", + startTime: at(HOUR_MS), + }), + sess("f2", 1, { visitorId: "v2", referrerHost: "t.co", country: "US", startTime: at(2 * HOUR_MS) }), + sess("f3", 1, { visitorId: "v3", referrerHost: "google.com", country: "DE", startTime: at(3 * HOUR_MS) }), + sess("f5", 1, { visitorId: "v1", startTime: at(5 * HOUR_MS) }), +] + +const seedFunnel = async (): Promise => { + const eventRows = FUNNEL_EVENTS.map( + (row) => + `(${quote(FUNNEL_ORG_ID)}, ${quote(row.sessionId)}, ${quote(row.ts)}, ${row.seq}, ${quote(row.type)}, ${quote(row.url)}, ${quote(row.message ?? "")}, map(), ${quote(row.visitorId)}, ${quote(row.userId)})`, + ).join(",\n") + await clickhouseExec( + `INSERT INTO session_events (OrgId, SessionId, Timestamp, Seq, Type, Url, Message, Attributes, VisitorId, UserId) VALUES\n${eventRows}`, + database, + ) + // f5 carries UserId so identity_links_mv links v1 → u1. + const sessionRows = FUNNEL_SESSIONS.map( + (row) => + `(${quote(FUNNEL_ORG_ID)}, ${quote(row.sessionId)}, ${quote(row.startTime)}, ${row.version}, ${quote(row.visitorId)}, ${quote(row.sessionId === "f5" ? "u1" : "")}, ${quote(row.referrerHost)}, ${quote(row.country)}, ${quote(row.utmSource)})`, + ).join(",\n") + await clickhouseExec( + `INSERT INTO session_replays (OrgId, SessionId, StartTime, Version, VisitorId, UserId, ReferrerHost, Country, UtmSource) VALUES\n${sessionRows}`, + database, + ) + await clickhouseExec( + `INSERT INTO product_events (OrgId, Timestamp, Source, UserId, Kind, EventName, ServiceName, Attributes) VALUES + (${quote(FUNNEL_ORG_ID)}, ${quote(at(6 * HOUR_MS))}, 'server', 'u1', 'custom', 'plan_started', 'maple-api', map('plan', 'startup')), + (${quote(FUNNEL_ORG_ID)}, ${quote(at(6 * HOUR_MS + MINUTE_MS))}, 'server', 'u9', 'custom', 'plan_started', 'maple-api', map('plan', 'free'))`, + database, + ) +} + +const REFERRAL_STEPS: ReadonlyArray = [ + { kind: "session", dimension: "referrerHost", value: "t.co" }, + { kind: "page", pagePath: "/pricing" }, + { kind: "event", eventName: "plan_started" }, +] +const PRODUCT_STEPS: ReadonlyArray = [ + { kind: "page", pagePath: "/pricing", host: "maple.dev" }, + { kind: "event", eventName: "signup_started" }, + { kind: "event", eventName: "plan_started", attributeEquals: { plan: "startup" } }, +] + +const funnelCounts = async (opts: CH.ProductEventsFunnelOpts): Promise> => { + const rows = await runJson(CH.compile(CH.productEventsFunnelQuery(opts), funnelWindow).sql) + return rows.map((row) => Number(row.count)) +} + +const breakdownRows = async ( + opts: CH.ProductEventsFunnelBreakdownOpts, +): Promise> => { + const rows = await runJson(CH.compile(CH.productEventsFunnelBreakdownQuery(opts), funnelWindow).sql) + return rows.map((row) => [row.group, Number(row.step), Number(row.count)]) +} + +describe.skipIf(!clickhouseE2eEnabled)("product events funnels", () => { + beforeAll(async () => { + // Shares the database (and its migrations) with the parity suite above; the + // seed is a different org so neither suite can see the other's rows. + await seedFunnel() + }, 60_000) + + it("seeds land in product_events and identity_links", async () => { + const [events, links] = await Promise.all([ + runJson(`SELECT count() AS n FROM product_events WHERE OrgId = ${quote(FUNNEL_ORG_ID)}`), + runJson( + `SELECT VisitorId, UserId FROM identity_links WHERE OrgId = ${quote(FUNNEL_ORG_ID)} GROUP BY VisitorId, UserId`, + ), + ]) + assert.strictEqual( + Number(events[0]?.n), + FUNNEL_EVENTS.length + 2, + "browser rows via the MV plus two server rows", + ) + assert.deepStrictEqual(links, [{ VisitorId: "v1", UserId: "u1" }]) + }) + + it("stitches a referred anonymous visit to the same person's server-side event", async () => { + // u1 (via v1): entry H1 → /pricing H1+1m → plan_started H6. v2: entry → /pricing. + assert.deepStrictEqual( + await funnelCounts({ steps: REFERRAL_STEPS, keyBy: "person", windowSeconds: 86_400 }), + [2, 2, 1], + ) + }) + + it("cannot reach the server-side step on a visitor or session key", async () => { + assert.deepStrictEqual( + await funnelCounts({ steps: REFERRAL_STEPS, keyBy: "visitor", windowSeconds: 86_400 }), + [2, 2, 0], + ) + assert.deepStrictEqual( + await funnelCounts({ steps: REFERRAL_STEPS, keyBy: "session", windowSeconds: 86_400 }), + [2, 2, 0], + ) + }) + + it("enforces the window from the step-1 event", async () => { + // plan_started is 5h after the referred entry: inside a day, outside an hour. + assert.deepStrictEqual( + await funnelCounts({ steps: REFERRAL_STEPS, keyBy: "person", windowSeconds: 3_600 }), + [2, 2, 0], + ) + }) + + it("counts persons in step order and ignores later steps without a step-1 event", async () => { + // u1: /pricing → signup_started → plan_started(startup) = 3. v2, v3: /pricing = 1. + // u9: plan_started only, no step 1 → 0. + assert.deepStrictEqual( + await funnelCounts({ steps: PRODUCT_STEPS, keyBy: "person", windowSeconds: 86_400 }), + [3, 1, 1], + ) + }) + + it("narrows the population by person when a filter is set", async () => { + // country=DE keeps f1 (→ u1) and f3 (v3); v2 (US) drops out entirely. + assert.deepStrictEqual( + await funnelCounts({ + steps: PRODUCT_STEPS, + keyBy: "person", + windowSeconds: 86_400, + filters: { country: "DE" }, + }), + [2, 1, 1], + ) + }) + + it("breaks a funnel down by a session dimension read through the person's sessions", async () => { + assert.deepStrictEqual( + await breakdownRows({ + steps: PRODUCT_STEPS, + keyBy: "person", + windowSeconds: 86_400, + breakdownBy: "referrerHost", + }), + [ + ["google.com", 1, 1], + ["google.com", 2, 0], + ["google.com", 3, 0], + ["t.co", 1, 2], + ["t.co", 2, 1], + ["t.co", 3, 1], + ], + ) + }) + + it("breaks a funnel down by an event attribute", async () => { + assert.deepStrictEqual( + await breakdownRows({ + steps: [{ kind: "event", eventName: "plan_started" }], + keyBy: "user", + windowSeconds: 60, + breakdownBy: "attribute:plan", + }), + [ + ["free", 1, 1], + ["startup", 1, 1], + ], + ) + }) + + it("lists event names with counts, sessions and persons", async () => { + const rows = await runJson(CH.compile(CH.productEventNamesQuery({ limit: 10 }), funnelWindow).sql) + assert.deepStrictEqual( + rows.map((row) => [row.eventName, row.kind, Number(row.count), Number(row.sessions), Number(row.persons)]), + [ + ["$pageview", "navigation", 6, 4, 4], + ["plan_started", "custom", 2, 0, 2], + ["signup_started", "custom", 1, 1, 1], + ], + ) + }) +}) diff --git a/lib/clickhouse-builder/docs/expressions.md b/lib/clickhouse-builder/docs/expressions.md index 6c449b907..9c95cd8d4 100644 --- a/lib/clickhouse-builder/docs/expressions.md +++ b/lib/clickhouse-builder/docs/expressions.md @@ -113,6 +113,19 @@ The `*If` family takes a `Condition` as its last argument: CH.quantile(0.95)($.DurationMs) // quantile(0.95)(DurationMs) ``` +So are the parametric funnel aggregates — the window / pattern is a parameter, +the timestamp and step conditions are the arguments: + +```ts +CH.windowFunnel(3600)($.Timestamp, $.Name.eq("view"), $.Name.eq("signup")) +// windowFunnel(3600)(Timestamp, Name = 'view', Name = 'signup') +CH.windowFunnel(3600, "strict_order")($.Timestamp, …) +CH.sequenceMatch("(?1)(?t<3600)(?2)")($.Timestamp, $.Name.eq("view"), $.Name.eq("signup")) +``` + +`windowFunnel` takes `Date`, `DateTime` or an unsigned integer for the timestamp +(not `DateTime64`) and the window is in that column's unit. + _(Backed by `docs/expressions.md > Conditional aggregation`.)_ ## Conditionals diff --git a/lib/clickhouse-builder/src/ch/core-dsl.test.ts b/lib/clickhouse-builder/src/ch/core-dsl.test.ts index c405256a8..54c2fa429 100644 --- a/lib/clickhouse-builder/src/ch/core-dsl.test.ts +++ b/lib/clickhouse-builder/src/ch/core-dsl.test.ts @@ -195,6 +195,46 @@ describe("expression functions", () => { }) }) +// Parametric aggregates + +describe("parametric aggregates", () => { + it("compiles windowFunnel with the window as a parameter and the conditions as arguments", () => { + const q = CH.from(TestTable) + .select(($) => ({ + id: $.Id, + level: CH.windowFunnel(3600)($.Timestamp, $.Name.eq("a"), $.Name.eq("b"), $.Value.gt(1)), + })) + .groupBy("id") + const { sql } = compileCH(q, {}) + expect(sql).toContain("windowFunnel(3600)(Timestamp, Name = 'a', Name = 'b', Value > 1) AS level") + }) + + it("compiles windowFunnel with a mode", () => { + const q = CH.from(TestTable).select(($) => ({ + level: CH.windowFunnel(86400, "strict_order")($.Timestamp, $.Name.eq("a"), $.Name.eq("b")), + })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("windowFunnel(86400, 'strict_order')(Timestamp, Name = 'a', Name = 'b') AS level") + }) + + it("windowFunnel refuses an empty condition list", () => { + const q = CH.from(TestTable).select(($) => ({ level: CH.windowFunnel(60)($.Timestamp) })) + expect(() => compileCH(q, {})).toThrow(/at least one condition/) + }) + + it("compiles sequenceMatch with the pattern as a parameter", () => { + const q = CH.from(TestTable).select(($) => ({ + matched: CH.sequenceMatch("(?1)(?t<3600)(?2)")($.Timestamp, $.Name.eq("a"), $.Name.eq("b")), + })) + const { sql } = compileCH(q, {}) + expect(sql).toContain("sequenceMatch('(?1)(?t<3600)(?2)')(Timestamp, Name = 'a', Name = 'b') AS matched") + }) + + it("sequenceMatch refuses a pattern that could break out of the literal", () => { + expect(() => CH.sequenceMatch("(?1)'; DROP")).toThrow(/quotes/) + }) +}) + // Condition combinators describe("condition combinators", () => { diff --git a/lib/clickhouse-builder/src/ch/functions/aggregate.ts b/lib/clickhouse-builder/src/ch/functions/aggregate.ts index 0968c6839..ffbe7e59f 100644 --- a/lib/clickhouse-builder/src/ch/functions/aggregate.ts +++ b/lib/clickhouse-builder/src/ch/functions/aggregate.ts @@ -92,3 +92,56 @@ export function groupUniqArrayIf(maxSize: number) { ), ) } + +/** The optional `windowFunnel` matching modes — see the ClickHouse docs. */ +export type WindowFunnelMode = "strict_order" | "strict_deduplication" | "strict_increase" + +/** + * `windowFunnel(window[, mode])(timestamp, cond1, cond2, …)` — the ClickHouse + * funnel aggregate: per group, the length of the longest prefix of + * `cond1..condN` that occurred in that order within `window` of the `cond1` + * event. + * + * `window` is in the unit of `timestamp` — for `DateTime`/`DateTime64` columns + * that is seconds, so callers pass `windowSeconds`. Ordering within a group + * happens inside the aggregate; no `ORDER BY` is needed on the input. + * + * Curried like {@link quantile}: the window and mode are *parameters* of the + * aggregate, the timestamp and conditions are its arguments. + */ +export function windowFunnel(window: number, mode?: WindowFunnelMode) { + const params = mode === undefined ? `${Math.round(window)}` : `${Math.round(window)}, '${mode}'` + return (timestamp: Expr, ...conditions: ReadonlyArray): Expr => { + if (conditions.length === 0) { + throw new Error("windowFunnel requires at least one condition") + } + const args = [timestamp.toFragment(), ...conditions.map((c) => c.toFragment())] + .map(compile) + .join(", ") + return makeExpr(raw(`windowFunnel(${params})(${args})`)) + } +} + +/** + * `sequenceMatch(pattern)(timestamp, cond1, cond2, …)` — 1 when the events + * matching `cond1..condN` occur in the order the pattern describes + * (`'(?1)(?2)'`, `'(?1)(?t<3600)(?2)'`, …), else 0. ClickHouse returns a + * `UInt8`, exposed as an `Expr` for `sumIf`/`countIf`-style use. + * + * The pattern is embedded verbatim — it is ClickHouse's pattern grammar, not + * user input, so only quote-free literals are accepted. + */ +export function sequenceMatch(pattern: string) { + if (pattern.includes("'") || pattern.includes("\\")) { + throw new Error("sequenceMatch pattern must not contain quotes or backslashes") + } + return (timestamp: Expr, ...conditions: ReadonlyArray): Expr => { + if (conditions.length === 0) { + throw new Error("sequenceMatch requires at least one condition") + } + const args = [timestamp.toFragment(), ...conditions.map((c) => c.toFragment())] + .map(compile) + .join(", ") + return makeExpr(raw(`sequenceMatch('${pattern}')(${args})`)) + } +} diff --git a/lib/clickhouse-builder/src/ch/functions/index.ts b/lib/clickhouse-builder/src/ch/functions/index.ts index b62a954eb..653931ccb 100644 --- a/lib/clickhouse-builder/src/ch/functions/index.ts +++ b/lib/clickhouse-builder/src/ch/functions/index.ts @@ -21,6 +21,9 @@ export { argMin, argMax, argMaxMerge, + windowFunnel, + sequenceMatch, + type WindowFunnelMode, } from "./aggregate" export { diff --git a/lib/clickhouse-builder/src/ch/index.ts b/lib/clickhouse-builder/src/ch/index.ts index 238df8235..baf2ab725 100644 --- a/lib/clickhouse-builder/src/ch/index.ts +++ b/lib/clickhouse-builder/src/ch/index.ts @@ -79,6 +79,9 @@ export { argMin, argMax, argMaxMerge, + windowFunnel, + sequenceMatch, + type WindowFunnelMode, // String toString_ as toString, positionCaseInsensitive, diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 1390307a8..92db8c739 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -539,6 +539,381 @@ SELECT ORDER BY bucket ASC FORMAT JSON +-- builder:product-events:productEventNamesQuery:default [7c7336c7] +SELECT + EventName AS eventName, + Kind AS kind, + count() AS count, + uniqIf(SessionId, SessionId != '') AS sessions, + uniq(if(UserId != '', UserId, VisitorId)) AS persons + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY eventName, kind + ORDER BY count DESC, eventName ASC + LIMIT 100 + FORMAT JSON + +-- builder:product-events:productEventNamesQuery:filtered [29add6ad] +SELECT + EventName AS eventName, + Kind AS kind, + count() AS count, + uniqIf(SessionId, SessionId != '') AS sessions, + uniq(if(UserId != '', UserId, VisitorId)) AS persons + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND Host = 'maple.dev' + AND SessionId IN (SELECT + SessionId AS sessionId + FROM session_replays + WHERE OrgId = 'org_sql_catalog' + AND StartTime >= '2026-01-01 10:30:00' + AND StartTime <= '2026-01-03 14:15:00' + AND SessionId IN (SELECT + SessionId AS sessionId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND Kind = 'navigation' + AND PagePath = '/pricing' + GROUP BY sessionId) + AND ReferrerHost = 't.co' + AND Country = 'DE' + AND DeviceType = 'desktop' + AND BrowserName = 'Chrome' + AND OsName = 'macOS' + AND Language = 'en-US' + AND UtmSource = 'twitter' + AND UtmMedium = 'social' + AND UtmCampaign = 'launch' + AND VisitorIsNew = 1 + GROUP BY sessionId) + GROUP BY eventName, kind + ORDER BY count DESC, eventName ASC + LIMIT 100 + FORMAT JSON + +-- builder:product-events:productEventsFunnelBreakdownQuery:attribute-session-step [ac39fa69] +SELECT + group AS group, + arrayJoin([1, 2, 3, 4]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + group AS group, + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3), countIf(level >= 4)] AS counts, + countIf(level >= 1) AS entered + FROM (SELECT + key AS key, + windowFunnel(604800000)(ts, s1 = 1, s2 = 1, s3 = 1, s4 = 1) AS level, + argMinIf(dim, ts, dim != '') AS group + FROM ( +SELECT + UserId AS key, + toUInt64(toUnixTimestamp64Milli(StartTime)) AS ts, + 1 AS s1, + 0 AS s2, + 0 AS s3, + 0 AS s4, + '' AS dim + FROM session_replays AS s + WHERE OrgId = 'org_sql_catalog' + AND StartTime >= '2026-01-01 10:30:00' + AND StartTime <= '2026-01-03 14:15:00' + AND ReferrerHost = 'news.ycombinator.com' + AND UserId != '' +UNION ALL +SELECT + UserId AS key, + toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, + 0 AS s1, + toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s2, + toUInt8(EventName = 'signup_completed') AS s3, + toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s4, + Attributes['plan'] AS dim + FROM product_events AS e + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ((((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev') OR EventName = 'signup_completed') OR (EventName = 'plan_started' AND Attributes['plan'] = 'startup')) + AND UserId != '' +) AS funnel_events + GROUP BY key) AS levels + GROUP BY group + ORDER BY entered DESC, group ASC + LIMIT 5) AS groups + ORDER BY group ASC, step ASC + FORMAT JSON + +-- builder:product-events:productEventsFunnelBreakdownQuery:session-dimension [5df587f2] +SELECT + group AS group, + arrayJoin([1, 2, 3]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + group AS group, + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3)] AS counts, + countIf(level >= 1) AS entered + FROM (SELECT + key AS key, + windowFunnel(604800000)(ts, s1 = 1, s2 = 1, s3 = 1) AS level, + argMinIf(dim, ts, dim != '') AS group + FROM (SELECT + multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) AS key, + toUInt64(toUnixTimestamp64Milli(e.Timestamp)) AS ts, + toUInt8(((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev')) AS s1, + toUInt8(e.EventName = 'signup_completed') AS s2, + toUInt8((e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) AS s3, + coalesce(sd.Value, '') AS dim + FROM product_events AS e + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId + LEFT JOIN (SELECT + SessionId AS SessionId, + max(UtmSource) AS Value + FROM session_replays + WHERE OrgId = 'org_sql_catalog' + AND StartTime >= '2026-01-01 10:30:00' + AND StartTime <= '2026-01-03 14:15:00' + GROUP BY SessionId) AS sd ON e.SessionId = sd.SessionId + WHERE e.OrgId = 'org_sql_catalog' + AND e.Timestamp >= '2026-01-01 10:30:00' + AND e.Timestamp <= '2026-01-03 14:15:00' + AND ((((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev') OR e.EventName = 'signup_completed') OR (e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) + AND multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) != '') AS funnel_events + GROUP BY key) AS levels + GROUP BY group + ORDER BY entered DESC, group ASC + LIMIT 10) AS groups + ORDER BY group ASC, step ASC + FORMAT JSON + +-- builder:product-events:productEventsFunnelQuery:person [5c76c8ff] +SELECT + arrayJoin([1, 2, 3]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3)] AS counts + FROM (SELECT + key AS key, + windowFunnel(604800000)(ts, s1 = 1, s2 = 1, s3 = 1) AS level + FROM (SELECT + multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) AS key, + toUInt64(toUnixTimestamp64Milli(e.Timestamp)) AS ts, + toUInt8(((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev')) AS s1, + toUInt8(e.EventName = 'signup_completed') AS s2, + toUInt8((e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) AS s3 + FROM product_events AS e + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId + WHERE e.OrgId = 'org_sql_catalog' + AND e.Timestamp >= '2026-01-01 10:30:00' + AND e.Timestamp <= '2026-01-03 14:15:00' + AND ((((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev') OR e.EventName = 'signup_completed') OR (e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) + AND multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) != '') AS funnel_events + GROUP BY key) AS levels) AS totals + ORDER BY step ASC + FORMAT JSON + +-- builder:product-events:productEventsFunnelQuery:session-key [619f95db] +SELECT + arrayJoin([1, 2, 3]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3)] AS counts + FROM (SELECT + key AS key, + windowFunnel(1800000)(ts, s1 = 1, s2 = 1, s3 = 1) AS level + FROM (SELECT + SessionId AS key, + toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, + toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s1, + toUInt8(EventName = 'signup_completed') AS s2, + toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s3 + FROM product_events AS e + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ((((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev') OR EventName = 'signup_completed') OR (EventName = 'plan_started' AND Attributes['plan'] = 'startup')) + AND SessionId != '') AS funnel_events + GROUP BY key) AS levels) AS totals + ORDER BY step ASC + FORMAT JSON + +-- builder:product-events:productEventsFunnelQuery:session-step-filtered [2502c451] +SELECT + arrayJoin([1, 2, 3, 4]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3), countIf(level >= 4)] AS counts + FROM (SELECT + key AS key, + windowFunnel(604800000)(ts, s1 = 1, s2 = 1, s3 = 1, s4 = 1) AS level + FROM ( +SELECT + multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) AS key, + toUInt64(toUnixTimestamp64Milli(s.StartTime)) AS ts, + 1 AS s1, + 0 AS s2, + 0 AS s3, + 0 AS s4 + FROM session_replays AS s + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId + WHERE s.OrgId = 'org_sql_catalog' + AND s.StartTime >= '2026-01-01 10:30:00' + AND s.StartTime <= '2026-01-03 14:15:00' + AND s.ReferrerHost = 'news.ycombinator.com' + AND multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) != '' + AND multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) IN (SELECT + multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) AS key + FROM session_replays AS s + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId + WHERE s.OrgId = 'org_sql_catalog' + AND s.StartTime >= '2026-01-01 10:30:00' + AND s.StartTime <= '2026-01-03 14:15:00' + AND s.SessionId IN (SELECT + SessionId AS sessionId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND Kind = 'navigation' + AND Host = 'maple.dev' + AND PagePath = '/pricing' + GROUP BY sessionId) + AND s.ReferrerHost = 't.co' + AND s.Country = 'DE' + AND s.DeviceType = 'desktop' + AND s.BrowserName = 'Chrome' + AND s.OsName = 'macOS' + AND s.Language = 'en-US' + AND s.UtmSource = 'twitter' + AND s.UtmMedium = 'social' + AND s.UtmCampaign = 'launch' + AND s.VisitorIsNew = 1 + GROUP BY key) +UNION ALL +SELECT + multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) AS key, + toUInt64(toUnixTimestamp64Milli(e.Timestamp)) AS ts, + 0 AS s1, + toUInt8(((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev')) AS s2, + toUInt8(e.EventName = 'signup_completed') AS s3, + toUInt8((e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) AS s4 + FROM product_events AS e + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId + WHERE e.OrgId = 'org_sql_catalog' + AND e.Timestamp >= '2026-01-01 10:30:00' + AND e.Timestamp <= '2026-01-03 14:15:00' + AND ((((e.Kind = 'navigation' AND e.PagePath = '/pricing') AND e.Host = 'maple.dev') OR e.EventName = 'signup_completed') OR (e.EventName = 'plan_started' AND e.Attributes['plan'] = 'startup')) + AND multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) != '' + AND multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) IN (SELECT + multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) AS key + FROM session_replays AS s + LEFT JOIN (SELECT + VisitorId AS VisitorId, + argMin(UserId, FirstSeen) AS UserId + FROM identity_links + WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId + WHERE s.OrgId = 'org_sql_catalog' + AND s.StartTime >= '2026-01-01 10:30:00' + AND s.StartTime <= '2026-01-03 14:15:00' + AND s.SessionId IN (SELECT + SessionId AS sessionId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND Kind = 'navigation' + AND Host = 'maple.dev' + AND PagePath = '/pricing' + GROUP BY sessionId) + AND s.ReferrerHost = 't.co' + AND s.Country = 'DE' + AND s.DeviceType = 'desktop' + AND s.BrowserName = 'Chrome' + AND s.OsName = 'macOS' + AND s.Language = 'en-US' + AND s.UtmSource = 'twitter' + AND s.UtmMedium = 'social' + AND s.UtmCampaign = 'launch' + AND s.VisitorIsNew = 1 + GROUP BY key) +) AS funnel_events + GROUP BY key) AS levels) AS totals + ORDER BY step ASC + FORMAT JSON + +-- builder:product-events:productEventsFunnelQuery:visitor-session-step [1eb9e5d6] +SELECT + arrayJoin([1, 2, 3, 4]) AS step, + arrayElement(counts, step) AS count + FROM (SELECT + [countIf(level >= 1), countIf(level >= 2), countIf(level >= 3), countIf(level >= 4)] AS counts + FROM (SELECT + key AS key, + windowFunnel(3600000)(ts, s1 = 1, s2 = 1, s3 = 1, s4 = 1) AS level + FROM ( +SELECT + VisitorId AS key, + toUInt64(toUnixTimestamp64Milli(StartTime)) AS ts, + 1 AS s1, + 0 AS s2, + 0 AS s3, + 0 AS s4 + FROM session_replays AS s + WHERE OrgId = 'org_sql_catalog' + AND StartTime >= '2026-01-01 10:30:00' + AND StartTime <= '2026-01-03 14:15:00' + AND ReferrerHost = 'news.ycombinator.com' + AND VisitorId != '' +UNION ALL +SELECT + VisitorId AS key, + toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, + 0 AS s1, + toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s2, + toUInt8(EventName = 'signup_completed') AS s3, + toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s4 + FROM product_events AS e + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ((((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev') OR EventName = 'signup_completed') OR (EventName = 'plan_started' AND Attributes['plan'] = 'startup')) + AND VisitorId != '' +) AS funnel_events + GROUP BY key) AS levels) AS totals + ORDER BY step ASC + FORMAT JSON + -- builder:service-map-rollup:serviceMapEdgesExistingHoursSQL:default [6a2a284a] SELECT toUnixTimestamp(Hour) AS hourTs @@ -2349,7 +2724,7 @@ SELECT LIMIT 50 FORMAT JSON --- builder:web-analytics:webAnalyticsBreakdownsQuery:all-dimensions-filtered-rollup [4fc9ae27] +-- builder:web-analytics:webAnalyticsBreakdownsQuery:all-dimensions-filtered-rollup [87c76923] SELECT ReferrerHost AS name, uniq(SessionId) AS count, @@ -2360,7 +2735,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2392,7 +2767,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2424,7 +2799,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2456,7 +2831,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2488,7 +2863,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2520,7 +2895,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2552,7 +2927,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2584,7 +2959,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2616,7 +2991,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2648,7 +3023,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2680,7 +3055,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -2712,7 +3087,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3068,13 +3443,13 @@ SELECT LIMIT 100 FORMAT JSON --- builder:web-analytics:webAnalyticsPagesQuery:default-rollup [89af71c3] +-- builder:web-analytics:webAnalyticsPagesQuery:default-rollup [db25069a] SELECT Host AS host, PagePath AS pagePath, count() AS pageViews, uniq(SessionId) AS sessions - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3110,13 +3485,13 @@ SELECT LIMIT 100 FORMAT JSON --- builder:web-analytics:webAnalyticsPagesQuery:semi-joined-rollup [12fc3e6e] +-- builder:web-analytics:webAnalyticsPagesQuery:semi-joined-rollup [95c72121] SELECT Host AS host, PagePath AS pagePath, count() AS pageViews, uniq(SessionId) AS sessions - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3153,13 +3528,13 @@ SELECT LIMIT 100 FORMAT JSON --- builder:web-analytics:webAnalyticsPagesQuery:url-filtered-rollup [90a8dfda] +-- builder:web-analytics:webAnalyticsPagesQuery:url-filtered-rollup [d0bf84d1] SELECT Host AS host, PagePath AS pagePath, count() AS pageViews, uniq(SessionId) AS sessions - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3185,12 +3560,12 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:web-analytics:webAnalyticsPageviewsTimeseriesQuery:default-rollup [0e7c3b62] +-- builder:web-analytics:webAnalyticsPageviewsTimeseriesQuery:default-rollup [f5af91eb] SELECT toStartOfInterval(Timestamp, INTERVAL 3600 SECOND) AS bucket, count() AS pageViews, uniq(SessionId) AS sessions - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3222,12 +3597,12 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:web-analytics:webAnalyticsPageviewsTimeseriesQuery:semi-joined-rollup [f3673589] +-- builder:web-analytics:webAnalyticsPageviewsTimeseriesQuery:semi-joined-rollup [6d125446] SELECT toStartOfInterval(Timestamp, INTERVAL 3600 SECOND) AS bucket, count() AS pageViews, uniq(SessionId) AS sessions - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' @@ -3307,7 +3682,7 @@ SELECT AND VisitorIsNew = 1 FORMAT JSON --- builder:web-analytics:webAnalyticsSummaryQuery:filtered-rollup [3746231e] +-- builder:web-analytics:webAnalyticsSummaryQuery:filtered-rollup [2662de3d] SELECT uniqIf(VisitorId, VisitorId != '') AS visitors, uniq(SessionId) AS sessions, @@ -3321,7 +3696,7 @@ SELECT AND StartTime <= '2026-01-03 14:15:00' AND SessionId IN (SELECT SessionId AS sessionId - FROM web_events + FROM product_events WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index 68cf4455f..e3ab52a15 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -60,7 +60,7 @@ const window = { orgId: ORG_ID, startTime: START_TIME, endTime: END_TIME } // forces the `SessionId IN (SELECT …)` semi-join branch, which is a whole // second SQL shape that no unfiltered fixture reaches. // -// Every one is emitted TWICE, once per page-view source. `useWebEvents` is a +// Every one is emitted TWICE, once per page-view source. `useProductEvents` is a // routing predicate, and `routeCoverage()` in sql-catalog.ts requires a // predicate to be exercised both ways — but the stronger reason is that the two // paths are required to return identical numbers, so both SQL shapes belong in @@ -84,65 +84,174 @@ const WEB_ANALYTICS_ALL_FILTERS = { const webAnalyticsVariants = ( name: string, label: string, - compile: (useWebEvents: boolean) => CompiledQuery, + compile: (useProductEvents: boolean) => CompiledQuery, ): ReadonlyArray => [ { module: "web-analytics", name, label, compile: () => compile(false) }, { module: "web-analytics", name, label: `${label}-rollup`, compile: () => compile(true) }, ] const webAnalyticsFixtures: ReadonlyArray = [ - ...webAnalyticsVariants("webAnalyticsSummaryQuery", "default", (useWebEvents) => - CH.compile(CH.webAnalyticsSummaryQuery({ useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsSummaryQuery", "default", (useProductEvents) => + CH.compile(CH.webAnalyticsSummaryQuery({ useProductEvents }), window), ), - ...webAnalyticsVariants("webAnalyticsSummaryQuery", "filtered", (useWebEvents) => - CH.compile(CH.webAnalyticsSummaryQuery({ ...WEB_ANALYTICS_ALL_FILTERS, useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsSummaryQuery", "filtered", (useProductEvents) => + CH.compile(CH.webAnalyticsSummaryQuery({ ...WEB_ANALYTICS_ALL_FILTERS, useProductEvents }), window), ), - ...webAnalyticsVariants("webAnalyticsTimeseriesQuery", "default", (useWebEvents) => - CH.compile(CH.webAnalyticsTimeseriesQuery({ bucketSeconds: 3600, useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsTimeseriesQuery", "default", (useProductEvents) => + CH.compile(CH.webAnalyticsTimeseriesQuery({ bucketSeconds: 3600, useProductEvents }), window), ), - ...webAnalyticsVariants("webAnalyticsPageviewsTimeseriesQuery", "default", (useWebEvents) => - CH.compile(CH.webAnalyticsPageviewsTimeseriesQuery({ bucketSeconds: 3600, useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsPageviewsTimeseriesQuery", "default", (useProductEvents) => + CH.compile(CH.webAnalyticsPageviewsTimeseriesQuery({ bucketSeconds: 3600, useProductEvents }), window), ), // Forces the semi-join: `referrerHost` is a session_replays-only dimension, // so the page-view source has to narrow through a subquery to honour it. - ...webAnalyticsVariants("webAnalyticsPageviewsTimeseriesQuery", "semi-joined", (useWebEvents) => + ...webAnalyticsVariants("webAnalyticsPageviewsTimeseriesQuery", "semi-joined", (useProductEvents) => CH.compile( CH.webAnalyticsPageviewsTimeseriesQuery({ bucketSeconds: 3600, referrerHost: "t.co", visitorType: "returning", - useWebEvents, + useProductEvents, }), window, ), ), - ...webAnalyticsVariants("webAnalyticsPagesQuery", "default", (useWebEvents) => - CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsPagesQuery", "default", (useProductEvents) => + CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, useProductEvents }), window), ), // host/pagePath filter directly off the page-view source — deliberately NOT // through the semi-join, so the 82% of sessions with no analytics block still count. - ...webAnalyticsVariants("webAnalyticsPagesQuery", "url-filtered", (useWebEvents) => - CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, host: "maple.dev", useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsPagesQuery", "url-filtered", (useProductEvents) => + CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, host: "maple.dev", useProductEvents }), window), ), - ...webAnalyticsVariants("webAnalyticsPagesQuery", "semi-joined", (useWebEvents) => - CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, country: "DE", useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsPagesQuery", "semi-joined", (useProductEvents) => + CH.compile(CH.webAnalyticsPagesQuery({ limit: 100, country: "DE", useProductEvents }), window), ), - ...webAnalyticsVariants("webAnalyticsBreakdownsQuery", "default", (useWebEvents) => - CH.compileUnion(CH.webAnalyticsBreakdownsQuery({ useWebEvents }), window), + ...webAnalyticsVariants("webAnalyticsBreakdownsQuery", "default", (useProductEvents) => + CH.compileUnion(CH.webAnalyticsBreakdownsQuery({ useProductEvents }), window), ), // Every dimension selected at once: each branch must exclude its own filter, // so this is the fixture that would catch a branch that forgot to. On the // rollup variant it is also the one that shows the navigation semi-join being // inlined into all twelve branches — the shape the rollup exists to make cheap. - ...webAnalyticsVariants("webAnalyticsBreakdownsQuery", "all-dimensions-filtered", (useWebEvents) => + ...webAnalyticsVariants("webAnalyticsBreakdownsQuery", "all-dimensions-filtered", (useProductEvents) => CH.compileUnion( - CH.webAnalyticsBreakdownsQuery({ ...WEB_ANALYTICS_ALL_FILTERS, useWebEvents }), + CH.webAnalyticsBreakdownsQuery({ ...WEB_ANALYTICS_ALL_FILTERS, useProductEvents }), window, ), ), ] +// Product-event funnel fixtures. The funnel SQL has four independent axes — +// person-key resolution (with or without the identity_links join), a session +// step 1 (UNION ALL of a session_replays branch), the population filter +// semi-join, and a breakdown dimension (event column vs session_replays join) — +// and each is a distinct SQL shape the analyzer has to see. +const FUNNEL_STEPS: ReadonlyArray = [ + { kind: "page", pagePath: "/pricing", host: "maple.dev" }, + { kind: "event", eventName: "signup_completed" }, + { kind: "event", eventName: "plan_started", attributeEquals: { plan: "startup" } }, +] +const REFERRAL_STEPS: ReadonlyArray = [ + { kind: "session", dimension: "referrerHost", value: "news.ycombinator.com" }, + ...FUNNEL_STEPS, +] + +const productEventsFixtures: ReadonlyArray = [ + { + module: "product-events", + name: "productEventsFunnelQuery", + label: "person", + compile: () => + CH.compile( + CH.productEventsFunnelQuery({ steps: FUNNEL_STEPS, keyBy: "person", windowSeconds: 7 * 86_400 }), + window, + ), + }, + { + module: "product-events", + name: "productEventsFunnelQuery", + label: "session-step-filtered", + compile: () => + CH.compile( + CH.productEventsFunnelQuery({ + steps: REFERRAL_STEPS, + keyBy: "person", + windowSeconds: 7 * 86_400, + filters: WEB_ANALYTICS_ALL_FILTERS, + }), + window, + ), + }, + { + module: "product-events", + name: "productEventsFunnelQuery", + label: "visitor-session-step", + compile: () => + CH.compile( + CH.productEventsFunnelQuery({ steps: REFERRAL_STEPS, keyBy: "visitor", windowSeconds: 3_600 }), + window, + ), + }, + { + module: "product-events", + name: "productEventsFunnelQuery", + label: "session-key", + compile: () => + CH.compile( + CH.productEventsFunnelQuery({ steps: FUNNEL_STEPS, keyBy: "session", windowSeconds: 1_800 }), + window, + ), + }, + { + module: "product-events", + name: "productEventsFunnelBreakdownQuery", + label: "session-dimension", + compile: () => + CH.compile( + CH.productEventsFunnelBreakdownQuery({ + steps: FUNNEL_STEPS, + keyBy: "person", + windowSeconds: 7 * 86_400, + breakdownBy: "utmSource", + limit: 10, + }), + window, + ), + }, + { + module: "product-events", + name: "productEventsFunnelBreakdownQuery", + label: "attribute-session-step", + compile: () => + CH.compile( + CH.productEventsFunnelBreakdownQuery({ + steps: REFERRAL_STEPS, + keyBy: "user", + windowSeconds: 7 * 86_400, + breakdownBy: "attribute:plan", + limit: 5, + }), + window, + ), + }, + { + module: "product-events", + name: "productEventNamesQuery", + label: "default", + compile: () => CH.compile(CH.productEventNamesQuery({ limit: 100 }), window), + }, + { + module: "product-events", + name: "productEventNamesQuery", + label: "filtered", + compile: () => + CH.compile(CH.productEventNamesQuery({ filters: WEB_ANALYTICS_ALL_FILTERS, limit: 100 }), window), + }, +] + export const builderFixtures: ReadonlyArray = [ + ...productEventsFixtures, // Session replay fixtures used by the replay routes. { module: "session-replays", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 8d74dde75..cbe5e6630 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -169,8 +169,32 @@ export { type WebAnalyticsPagesOutput, type WebAnalyticsBreakdownsOpts, type WebAnalyticsBreakdownsOutput, + type ProductEventsFilters, } from "./queries/web-analytics" +// Queries — Product events (funnels over `product_events`) +export { + productEventsFunnelQuery, + productEventsFunnelRowSchema, + productEventsFunnelBreakdownQuery, + productEventsFunnelBreakdownRowSchema, + productEventNamesQuery, + productEventNamesRowSchema, + ProductEventsFunnelError, + FUNNEL_MAX_STEPS, + FUNNEL_BREAKDOWN_MAX_GROUPS, + type FunnelStep, + type FunnelKeyBy, + type FunnelSessionDimension, + type FunnelBreakdownBy, + type ProductEventsFunnelOpts, + type ProductEventsFunnelOutput, + type ProductEventsFunnelBreakdownOpts, + type ProductEventsFunnelBreakdownOutput, + type ProductEventNamesOpts, + type ProductEventNamesOutput, +} from "./queries/product-events" + // Queries — Services export { serviceOverviewQuery, diff --git a/packages/query-engine/src/ch/queries/product-events.test.ts b/packages/query-engine/src/ch/queries/product-events.test.ts new file mode 100644 index 000000000..20c0ae550 --- /dev/null +++ b/packages/query-engine/src/ch/queries/product-events.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from "vitest" +import { compileCH } from "@maple-dev/clickhouse-builder" +import { + productEventsFunnelQuery, + productEventsFunnelBreakdownQuery, + productEventNamesQuery, + ProductEventsFunnelError, + type FunnelStep, +} from "./product-events" + +const params = { orgId: "org_1", startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00" } + +const STEPS: ReadonlyArray = [ + { kind: "page", pagePath: "/pricing", host: "maple.dev" }, + { kind: "event", eventName: "signup_completed" }, + { kind: "event", eventName: "plan_started", attributeEquals: { plan: "startup" } }, +] +const REFERRAL: FunnelStep = { kind: "session", dimension: "referrerHost", value: "news.ycombinator.com" } + +const oneLine = (sql: string): string => sql.replace(/\s+/g, " ") + +// productEventsFunnelQuery +// +// One `windowFunnel` per person over the rows matching any step, then +// `countIf(level >= n)` per step, unpacked to `{ step, count }` rows. + +describe("productEventsFunnelQuery", () => { + it("scopes every table it reads to the org and derives an org-scoped result", () => { + const compiled = compileCH( + productEventsFunnelQuery({ steps: [REFERRAL, ...STEPS], keyBy: "person", windowSeconds: 86_400 }), + params, + ) + expect(compiled.tenantScope).toBe("org") + // product_events, session_replays (session step) and identity_links (person key). + expect(compiled.sql).toContain("FROM product_events AS e") + expect(compiled.sql).toContain("FROM session_replays AS s") + expect(compiled.sql).toContain("FROM identity_links") + expect(compiled.sql.match(/OrgId = 'org_1'/g)?.length).toBeGreaterThanOrEqual(4) + }) + + it("emits one windowFunnel condition per step and one output row per step", () => { + const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), params) + // The window is passed in the timestamp's unit — epoch milliseconds — since + // windowFunnel does not accept DateTime64. + expect(sql).toContain("windowFunnel(3600000)(ts, s1 = 1, s2 = 1, s3 = 1) AS level") + expect(sql).toContain("toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts") + expect(sql).toContain("[countIf(level >= 1), countIf(level >= 2), countIf(level >= 3)] AS counts") + expect(sql).toContain("arrayJoin([1, 2, 3]) AS step") + expect(sql).toContain("arrayElement(counts, step) AS count") + expect(sql).toContain("ORDER BY step ASC") + }) + + it("projects each step as a flag and only reads rows matching some step", () => { + const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), params) + expect(sql).toContain("toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s1") + expect(sql).toContain("toUInt8(EventName = 'signup_completed') AS s2") + expect(sql).toContain("toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s3") + expect(oneLine(sql)).toContain( + "AND ((((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev') OR EventName = 'signup_completed') OR (EventName = 'plan_started' AND Attributes['plan'] = 'startup'))", + ) + }) + + it("keys by the raw column for visitor / user / session and drops empty keys", () => { + const visitor = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), params).sql + expect(visitor).toContain("VisitorId AS key") + expect(visitor).toContain("AND VisitorId != ''") + expect(visitor).not.toContain("identity_links") + + const user = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "user", windowSeconds: 60 }), params).sql + expect(user).toContain("UserId AS key") + expect(user).toContain("AND UserId != ''") + + const session = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "session", windowSeconds: 60 }), params).sql + expect(session).toContain("SessionId AS key") + expect(session).toContain("AND SessionId != ''") + }) + + it("stitches the person key through identity_links aggregated per visitor", () => { + const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), params) + expect(oneLine(sql)).toContain( + "LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId FROM identity_links WHERE OrgId = 'org_1' GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId", + ) + expect(sql).toContain( + "multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) AS key", + ) + }) + + it("turns a session step 1 into a UNION ALL branch of session_replays entries", () => { + const { sql } = compileCH( + productEventsFunnelQuery({ steps: [REFERRAL, ...STEPS], keyBy: "visitor", windowSeconds: 86_400 }), + params, + ) + expect(sql).toContain("UNION ALL") + // The session branch: s1 = 1, every other step 0, at the session's StartTime. + expect(oneLine(sql)).toContain( + "SELECT VisitorId AS key, toUInt64(toUnixTimestamp64Milli(StartTime)) AS ts, 1 AS s1, 0 AS s2, 0 AS s3, 0 AS s4 FROM session_replays AS s", + ) + expect(sql).toContain("AND ReferrerHost = 'news.ycombinator.com'") + // The events branch never satisfies the session step. + expect(oneLine(sql)).toContain("SELECT VisitorId AS key, toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, 0 AS s1,") + expect(sql).toContain("windowFunnel(86400000)(ts, s1 = 1, s2 = 1, s3 = 1, s4 = 1) AS level") + }) + + it("has no session_replays branch without a session step", () => { + const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), params) + expect(sql).not.toContain("UNION ALL") + expect(sql).not.toContain("session_replays") + }) + + it("narrows the population by person, not by session, when filters are set", () => { + const { sql } = compileCH( + productEventsFunnelQuery({ + steps: STEPS, + keyBy: "person", + windowSeconds: 60, + filters: { country: "DE", pagePath: "/" }, + }), + params, + ) + const flat = oneLine(sql) + // The persons subquery resolves the key the same way as the events do… + expect(flat).toContain( + "IN (SELECT multiIf(s.UserId != '', s.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), s.VisitorId) AS key FROM session_replays AS s LEFT JOIN", + ) + // …applies the replays dimension directly… + expect(flat).toContain("AND s.Country = 'DE'") + // …and the page filter through the navigation semi-join on product_events. + expect(flat).toContain("AND s.SessionId IN (SELECT SessionId AS sessionId FROM product_events WHERE OrgId = 'org_1'") + expect(flat).toContain("AND Kind = 'navigation' AND PagePath = '/' GROUP BY sessionId)") + }) + + it("omits the population subquery when no filter is set", () => { + const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), params) + expect(sql).not.toContain(" IN (SELECT") + }) + + it("rejects funnels it cannot compile", () => { + expect(() => productEventsFunnelQuery({ steps: [], keyBy: "person", windowSeconds: 60 })).toThrow( + ProductEventsFunnelError, + ) + expect(() => + productEventsFunnelQuery({ steps: [STEPS[0]!, REFERRAL], keyBy: "person", windowSeconds: 60 }), + ).toThrow(/only valid as step 1/) + expect(() => productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 0 })).toThrow( + /windowSeconds/, + ) + expect(() => + productEventsFunnelQuery({ + steps: Array.from({ length: 11 }, () => STEPS[1]!), + keyBy: "person", + windowSeconds: 60, + }), + ).toThrow(/at most 10 steps/) + }) +}) + +// productEventsFunnelBreakdownQuery + +describe("productEventsFunnelBreakdownQuery", () => { + it("groups persons by the first non-empty dimension value and keeps the top N by step-1 count", () => { + const compiled = compileCH( + productEventsFunnelBreakdownQuery({ + steps: STEPS, + keyBy: "visitor", + windowSeconds: 3_600, + breakdownBy: "attribute:plan", + limit: 5, + }), + params, + ) + expect(compiled.tenantScope).toBe("org") + const flat = oneLine(compiled.sql) + expect(flat).toContain("Attributes['plan'] AS dim") + expect(flat).toContain("argMinIf(dim, ts, dim != '') AS group") + expect(flat).toContain("countIf(level >= 1) AS entered") + expect(flat).toContain("GROUP BY group ORDER BY entered DESC, group ASC LIMIT 5") + expect(flat).toContain("SELECT group AS group, arrayJoin([1, 2, 3]) AS step, arrayElement(counts, step) AS count") + expect(flat).toContain("ORDER BY group ASC, step ASC") + }) + + it("reads a session dimension through a per-session join on session_replays", () => { + const { sql } = compileCH( + productEventsFunnelBreakdownQuery({ + steps: STEPS, + keyBy: "visitor", + windowSeconds: 3_600, + breakdownBy: "utmSource", + }), + params, + ) + const flat = oneLine(sql) + expect(flat).toContain("coalesce(sd.Value, '') AS dim") + expect(flat).toContain( + "LEFT JOIN (SELECT SessionId AS SessionId, max(UtmSource) AS Value FROM session_replays WHERE OrgId = 'org_1'", + ) + expect(flat).toContain("AS sd ON e.SessionId = sd.SessionId") + expect(flat).toContain("LIMIT 10") + }) + + it("reads the dimension straight off the session row on the session-step branch", () => { + const { sql } = compileCH( + productEventsFunnelBreakdownQuery({ + steps: [REFERRAL, ...STEPS], + keyBy: "visitor", + windowSeconds: 3_600, + breakdownBy: "country", + }), + params, + ) + expect(oneLine(sql)).toContain("0 AS s4, Country AS dim FROM session_replays AS s") + }) + + it("uses the event Host without a join", () => { + const { sql } = compileCH( + productEventsFunnelBreakdownQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60, breakdownBy: "host" }), + params, + ) + expect(sql).toContain("Host AS dim") + expect(sql).not.toContain("session_replays") + }) + + it("bounds the group limit", () => { + expect(() => + productEventsFunnelBreakdownQuery({ + steps: STEPS, + keyBy: "visitor", + windowSeconds: 60, + breakdownBy: "host", + limit: 21, + }), + ).toThrow(/1\.\.20/) + }) +}) + +// productEventNamesQuery + +describe("productEventNamesQuery", () => { + it("lists names with counts, sessions and persons, most frequent first", () => { + const compiled = compileCH(productEventNamesQuery({ limit: 25 }), params) + expect(compiled.tenantScope).toBe("org") + const flat = oneLine(compiled.sql) + expect(flat).toContain( + "SELECT EventName AS eventName, Kind AS kind, count() AS count, uniqIf(SessionId, SessionId != '') AS sessions, uniq(if(UserId != '', UserId, VisitorId)) AS persons FROM product_events", + ) + expect(flat).toContain("WHERE OrgId = 'org_1'") + expect(flat).toContain("GROUP BY eventName, kind ORDER BY count DESC, eventName ASC LIMIT 25") + expect(flat).not.toContain("session_replays") + }) + + it("applies host directly and other filters through the session semi-join", () => { + const { sql } = compileCH( + productEventNamesQuery({ filters: { host: "maple.dev", referrerHost: "t.co", pagePath: "/pricing" } }), + params, + ) + const flat = oneLine(sql) + expect(flat).toContain("AND Host = 'maple.dev' AND SessionId IN (SELECT SessionId AS sessionId FROM session_replays") + expect(flat).toContain("AND ReferrerHost = 't.co'") + // pagePath narrows sessions through the navigation semi-join, not the events. + expect(flat).toContain("AND Kind = 'navigation' AND PagePath = '/pricing' GROUP BY sessionId)") + }) +}) diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts new file mode 100644 index 000000000..8789d57e0 --- /dev/null +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -0,0 +1,643 @@ +// Product events — funnels over `product_events`. +// +// `product_events` is the one time-sorted table every product event lands in: +// browser page views and `track()` calls via the `session_events` MV, backend +// and mobile events via `POST /v1/events`. Funnels step over it with +// `windowFunnel`, grouped by a per-person key, and every query here keeps the +// page-level filter surface (`ProductEventsFilters`) so the `/analytics` +// sidebar narrows a funnel exactly the way it narrows the page-view panels. +// +// The page-view queries themselves live in `web-analytics.ts` and read the +// same table (`useProductEvents`); this module owns everything funnel-shaped. + +import * as CH from "@maple-dev/clickhouse-builder/expr" +import { + param, + from, + fromQuery, + fromUnion, + unionAll, + inSubquery, + compileFnCall, +} from "@maple-dev/clickhouse-builder" +import type { CHQuery, ColumnAccessor, ColumnDefs, JoinedColumnAccessor } from "@maple-dev/clickhouse-builder" +import { Schema } from "effect" +import { ProductEvents, IdentityLinks, SessionReplays } from "../tables" +import { CHNumber } from "../schema" +import { + replaysWhere, + needsSessionSemiJoin, + type ProductEventsFilters, +} from "./web-analytics" + +export type { ProductEventsFilters } from "./web-analytics" + +// Local function helpers (generic per call site, so not `defineFn`) + +// arrayElement(arr, i) — 1-based, like ClickHouse. +function arrayElement(arr: CH.Expr>, index: CH.Expr): CH.Expr { + return compileFnCall("arrayElement", arr, index) +} + +// toUInt8(cond) — a condition as a projectable 0/1 column. +function flag(cond: CH.Condition): CH.Expr { + return compileFnCall("toUInt8", cond) +} + +// toUInt64(toUnixTimestamp64Milli(ts)) — `windowFunnel` accepts Date, DateTime +// and unsigned integers but NOT DateTime64, so both branches carry their +// timestamp as epoch milliseconds and the window is `windowSeconds * 1000`. +// Milliseconds rather than `toDateTime()` so two events in the same second +// (page load → track()) keep their real order instead of tying. +function epochMs(ts: CH.Expr): CH.Expr { + return CH.toUInt64(compileFnCall("toUnixTimestamp64Milli", ts)) +} + +// argMinIf(value, orderBy, cond) — the `value` on the earliest row matching `cond`. +function argMinIf(value: CH.Expr, orderBy: CH.Expr, cond: CH.Condition): CH.Expr { + return compileFnCall("argMinIf", value, orderBy, cond) +} + +// Every UNION ALL branch below is built from a different table, and the shape +// they share is this. `unionAll`'s type unification cannot see through the +// per-branch `select` callbacks (each infers its own literal type), so the +// branches are annotated to it explicitly. +type FunnelEventRow = { readonly [column: string]: unknown } +type FunnelBranch = CHQuery> + +// Public option types + +/** Which `session_replays` dimension a `session` step (or a breakdown) reads. */ +export type FunnelSessionDimension = "referrerHost" | "utmSource" | "utmMedium" | "utmCampaign" | "country" | "host" + +/** + * One funnel step. + * + * - `event`: a `track()` (or direct-ingested) event by name, optionally + * narrowed by `Attributes[k] = v` for every entry of `attributeEquals`. + * - `page`: a page view of `pagePath` (`Kind = 'navigation'`), optionally on + * one `host`. + * - `session`: "started a session with this acquisition dimension" — the + * referral / campaign entry point. Only valid as step 1: it is evaluated + * against `session_replays` and enters the funnel as a synthetic + * `$session_entry` event at the session's `StartTime`. + */ +export type FunnelStep = + | { + readonly kind: "event" + readonly eventName: string + readonly attributeEquals?: Readonly> + } + | { readonly kind: "page"; readonly pagePath: string; readonly host?: string } + | { readonly kind: "session"; readonly dimension: FunnelSessionDimension; readonly value: string } + +/** + * What a funnel counts. + * + * - `person`: `UserId` when the row carries one, else the `VisitorId`'s linked + * user from `identity_links`, else the `VisitorId` — so an anonymous marketing + * visit and the same person's post-signup (or server-side) events collapse + * into one person. + * - `visitor` / `user`: the raw column, non-empty. + * - `session`: `SessionId` — a per-session funnel; server events (no session) + * never take part. + */ +export type FunnelKeyBy = "person" | "visitor" | "user" | "session" + +export interface ProductEventsFunnelOpts { + /** 1–10 steps, in order. A `session` step may only appear first. */ + readonly steps: ReadonlyArray + readonly keyBy: FunnelKeyBy + /** The whole chain must complete within this many seconds of the step-1 event. */ + readonly windowSeconds: number + /** Sidebar filters — narrow the *population* to persons with a matching session. */ + readonly filters?: ProductEventsFilters +} + +/** + * How a breakdown groups persons: an acquisition dimension of the person's + * sessions, the event `Host`, or `Attributes[]` on the person's events. + * In every case the group is the first non-empty value seen for that person. + */ +export type FunnelBreakdownBy = FunnelSessionDimension | `attribute:${string}` + +export interface ProductEventsFunnelBreakdownOpts extends ProductEventsFunnelOpts { + readonly breakdownBy: FunnelBreakdownBy + /** Groups to keep, ranked by step-1 count. Default 10, max 20. */ + readonly limit?: number +} + +export interface ProductEventNamesOpts { + readonly filters?: ProductEventsFilters + /** Default 100. */ + readonly limit?: number +} + +// Row schemas + +export const productEventsFunnelRowSchema = Schema.Struct({ + step: CHNumber, + count: CHNumber, +}) +export type ProductEventsFunnelOutput = typeof productEventsFunnelRowSchema.Type + +export const productEventsFunnelBreakdownRowSchema = Schema.Struct({ + group: Schema.String, + step: CHNumber, + count: CHNumber, +}) +export type ProductEventsFunnelBreakdownOutput = typeof productEventsFunnelBreakdownRowSchema.Type + +export const productEventNamesRowSchema = Schema.Struct({ + eventName: Schema.String, + kind: Schema.String, + count: CHNumber, + sessions: CHNumber, + persons: CHNumber, +}) +export type ProductEventNamesOutput = typeof productEventNamesRowSchema.Type + +// Validation + +/** A funnel definition the builder cannot compile. Thrown, since builders are synchronous. */ +export class ProductEventsFunnelError extends Schema.TaggedError()( + "@maple/query-engine/ProductEventsFunnelError", + { + reason: Schema.Literals(["NoSteps", "TooManySteps", "SessionStepNotFirst", "InvalidWindow", "InvalidLimit"]), + message: Schema.String, + }, +) {} + +export const FUNNEL_MAX_STEPS = 10 +export const FUNNEL_BREAKDOWN_MAX_GROUPS = 20 + +function validate(opts: ProductEventsFunnelOpts): void { + if (opts.steps.length === 0) { + throw new ProductEventsFunnelError({ reason: "NoSteps", message: "a funnel needs at least one step" }) + } + if (opts.steps.length > FUNNEL_MAX_STEPS) { + throw new ProductEventsFunnelError({ + reason: "TooManySteps", + message: `a funnel has at most ${FUNNEL_MAX_STEPS} steps, got ${opts.steps.length}`, + }) + } + opts.steps.forEach((step, index) => { + if (step.kind === "session" && index !== 0) { + throw new ProductEventsFunnelError({ + reason: "SessionStepNotFirst", + message: `a session step is only valid as step 1, found one at step ${index + 1}`, + }) + } + }) + if (!Number.isFinite(opts.windowSeconds) || opts.windowSeconds <= 0) { + throw new ProductEventsFunnelError({ + reason: "InvalidWindow", + message: `windowSeconds must be a positive number, got ${String(opts.windowSeconds)}`, + }) + } +} + +// Shared pieces + +type EventsAccessor = ColumnAccessor +type ReplaysAccessor = ColumnAccessor +type IdentityAccessor = { + readonly SessionId: CH.Expr + readonly VisitorId: CH.Expr + readonly UserId: CH.Expr +} +// The `identity_links` join alias — a typed accessor when the join is declared +// statically, an open one under `OpenJoinQuery`. +type LinkAccessor = { readonly UserId: CH.Expr } | ColumnAccessor + +const LINK_ALIAS = "link" + +/** + * `identity_links` collapsed to one linked user per visitor. ReplacingMergeTree + * may still hold several rows per (visitor, user) and a visitor may have been + * linked to more than one user; `argMin(UserId, FirstSeen)` picks the user the + * visitor became *first*, which is the identity that anonymous rows + * chronologically precede — the one a conversion funnel wants. + */ +function identityLinksByVisitor() { + return from(IdentityLinks) + .select(($) => ({ VisitorId: $.VisitorId, UserId: CH.argMin($.UserId, $.FirstSeen) })) + .where(($) => [$.OrgId.eq(param.string("orgId"))]) + .groupBy("VisitorId") +} + +/** The person key for one row, per {@link FunnelKeyBy}. `link` is set only for `person`. */ +function personKey(keyBy: FunnelKeyBy, $: IdentityAccessor, link?: LinkAccessor): CH.Expr { + switch (keyBy) { + case "session": + return $.SessionId + case "visitor": + return $.VisitorId + case "user": + return $.UserId + case "person": { + // join_use_nulls=0 (the default) yields '' for an unmatched LEFT JOIN; + // coalesce covers a cluster that flips it on and yields NULL instead. + const linked = link ? compileFnCall("coalesce", link.UserId, CH.lit("")) : CH.lit("") + return CH.multiIf( + [ + [$.UserId.neq(""), $.UserId], + [linked.neq(""), linked], + ], + $.VisitorId, + ) + } + } +} + +/** Column of `session_replays` behind a session dimension. */ +function sessionDimensionColumn($: ReplaysAccessor, dimension: FunnelSessionDimension): CH.Expr { + switch (dimension) { + case "referrerHost": + return $.ReferrerHost + case "utmSource": + return $.UtmSource + case "utmMedium": + return $.UtmMedium + case "utmCampaign": + return $.UtmCampaign + case "country": + return $.Country + case "host": + return $.Host + } +} + +/** The step's predicate over a `product_events` row. `session` steps never match an event row. */ +function eventStepCondition($: EventsAccessor, step: FunnelStep): CH.Condition | undefined { + switch (step.kind) { + case "event": { + let cond = $.EventName.eq(step.eventName) + for (const [key, value] of Object.entries(step.attributeEquals ?? {})) { + cond = cond.and($.Attributes.get(key).eq(value)) + } + return cond + } + case "page": { + const cond = $.Kind.eq("navigation").and($.PagePath.eq(step.pagePath)) + return step.host === undefined ? cond : cond.and($.Host.eq(step.host)) + } + case "session": + return undefined + } +} + +/** True when any sidebar filter is set — i.e. the population must be narrowed. */ +function hasPopulationFilter(filters: ProductEventsFilters): boolean { + return needsSessionSemiJoin(filters) || filters.host !== undefined || filters.pagePath !== undefined +} + +/** + * `SELECT key FROM session_replays WHERE ` — the persons whose + * sessions match the filters, under the same key resolution as the events. + * + * Person-level rather than `SessionId IN (…)` on purpose: server-side rows have + * no `SessionId`, so a session semi-join would silently drop every backend + * step the moment a filter is active. `host` / `pagePath` reach here through + * `replaysWhere`'s navigation semi-join, which reads `product_events` — funnels + * require that table anyway. + */ +function matchingPersonsSubquery(keyBy: FunnelKeyBy, filters: ProductEventsFilters) { + const scoped = { ...filters, useProductEvents: true } + if (keyBy === "person") { + return from(SessionReplays, "s") + .leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (s, link) => s.VisitorId.eq(link.VisitorId)) + .select(($) => ({ key: personKey(keyBy, $, $[LINK_ALIAS]) })) + .where(($) => replaysWhere($, scoped)) + .groupBy("key") + } + return from(SessionReplays) + .select(($) => ({ key: personKey(keyBy, $) })) + .where(($) => replaysWhere($, scoped)) + .groupBy("key") +} + +const stepColumn = (index: number) => `s${index + 1}` + +/** `{ s1: …, s2: …, … }` — one projected flag column per step, in order. */ +function stepFlags( + steps: ReadonlyArray, + value: (step: FunnelStep, index: number) => CH.Expr, +): Record> { + return Object.fromEntries(steps.map((step, index) => [stepColumn(index), value(step, index)])) +} + +/** + * A query over `Cols` whose join map is left open. The branch builders attach + * joins conditionally (identity links for `person`, session dimensions for a + * breakdown), so the joined shape is not one static type; under an open map an + * alias resolves to untyped column refs and the builder only reads the aliases + * it declared. The `Output` is `{}` because `select` is the last call. + */ +type OpenJoinQuery = CHQuery> +type OpenJoinAccessor = JoinedColumnAccessor> + +interface FunnelPlan { + readonly opts: ProductEventsFunnelOpts + readonly filters: ProductEventsFilters + readonly sessionStep: Extract | undefined + /** Set when a breakdown wants a per-row dimension projected as `dim`. */ + readonly breakdownBy?: FunnelBreakdownBy +} + +/** + * The events branch: every `product_events` row in range that matches at least + * one step, projected to `(key, ts, s1..sN[, dim])`. + * + * Only rows matching *some* step are read — a funnel over three events does not + * scan every page view in the range. The step flags are computed here so the + * aggregate layer never sees `EventName`/`PagePath`/`Attributes` at all. + */ +function eventsBranch(plan: FunnelPlan): FunnelBranch { + const { opts, filters, breakdownBy } = plan + const keyBy = opts.keyBy + const dim = breakdownBy + // `host` is a session dimension too, but product_events carries it on every + // browser row, so it is read there rather than joined in. + const sessionDimension = + dim !== undefined && !dim.startsWith("attribute:") && dim !== "host" + ? (dim as FunnelSessionDimension) + : undefined + + // Per-session acquisition dimension for a session-dimension breakdown. + // `max()` so the v2 row's value beats a v1 row's ''. + const sessionDims = sessionDimension + ? from(SessionReplays) + .select(($) => ({ + SessionId: $.SessionId, + Value: CH.max_(sessionDimensionColumn($, sessionDimension)), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.StartTime.gte(param.dateTime("startTime")), + $.StartTime.lte(param.dateTime("endTime")), + ]) + .groupBy("SessionId") + : undefined + + // The joins differ by keyBy/breakdown, so the query is held under an open + // join map: joined aliases resolve to untyped column refs, and only the + // aliases actually declared above are ever read. + let base: OpenJoinQuery = from(ProductEvents, "e") + if (keyBy === "person") { + base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (e, link) => e.VisitorId.eq(link.VisitorId)) + } + if (sessionDims) { + base = base.leftJoinQuery(sessionDims, "sd", (e, sd) => e.SessionId.eq(sd.SessionId)) + } + + const dimExpr = ($: OpenJoinAccessor): CH.Expr | undefined => { + if (dim === undefined) return undefined + if (dim.startsWith("attribute:")) return $.Attributes.get(dim.slice("attribute:".length)) + if (dim === "host") return $.Host + return compileFnCall("coalesce", $.sd.Value, CH.lit("")) + } + + return base + .select(($) => { + const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) + const flags = stepFlags(opts.steps, (step) => { + const cond = eventStepCondition($, step) + return cond ? flag(cond) : CH.lit(0) + }) + const d = dimExpr($) + const row = { key, ts: epochMs($.Timestamp), ...flags } + return d ? { ...row, dim: d } : row + }) + .where(($) => { + const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) + const stepConditions = opts.steps + .map((step) => eventStepCondition($, step)) + .filter((cond): cond is CH.Condition => cond !== undefined) + const anyStep = stepConditions.reduce( + (acc, cond) => (acc ? acc.or(cond) : cond), + undefined, + ) + return [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + anyStep, + key.neq(""), + hasPopulationFilter(filters) ? inSubquery(key, matchingPersonsSubquery(keyBy, filters)) : undefined, + ] + }) +} + +/** + * The session-entry branch for a `session` step 1: one synthetic + * `$session_entry` row per matching session at its `StartTime`, with `s1 = 1` + * and every other step 0. Un-merged v1/v2 rows of one session yield two chain + * starts at the same instant, which `windowFunnel` treats as one. + */ +function sessionEntryBranch(plan: FunnelPlan, step: Extract): FunnelBranch { + const { opts, filters, breakdownBy } = plan + const keyBy = opts.keyBy + const dim = breakdownBy + const sessionDimension = + dim !== undefined && !dim.startsWith("attribute:") ? (dim as FunnelSessionDimension) : undefined + + let base: OpenJoinQuery = from(SessionReplays, "s") + if (keyBy === "person") { + base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (s, link) => s.VisitorId.eq(link.VisitorId)) + } + + return base + .select(($) => { + const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) + const flags = stepFlags(opts.steps, (_, index) => CH.lit(index === 0 ? 1 : 0)) + const row = { key, ts: epochMs($.StartTime), ...flags } + if (dim === undefined) return row + // An attribute breakdown has no value on a session row; the events + // branch supplies it. A session dimension is read straight off the row. + return { ...row, dim: sessionDimension ? sessionDimensionColumn($, sessionDimension) : CH.lit("") } + }) + .where(($) => { + const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) + return [ + $.OrgId.eq(param.string("orgId")), + $.StartTime.gte(param.dateTime("startTime")), + $.StartTime.lte(param.dateTime("endTime")), + sessionDimensionColumn($, step.dimension).eq(step.value), + key.neq(""), + hasPopulationFilter(filters) ? inSubquery(key, matchingPersonsSubquery(keyBy, filters)) : undefined, + ] + }) +} + +/** + * `SELECT key, windowFunnel(w)(ts, s1 = 1, …, sN = 1) AS level [, first dim AS group] + * FROM GROUP BY key` — one row per person with the + * deepest step they reached in order within the window. + */ +function levelsQuery(plan: FunnelPlan) { + const { opts } = plan + const events = eventsBranch(plan) + const source = plan.sessionStep + ? fromUnion(unionAll(sessionEntryBranch(plan, plan.sessionStep), events), "funnel_events") + : fromQuery(events, "funnel_events") + + return source + .select(($) => { + const ts = $.ts as CH.Expr + const conditions = opts.steps.map((_, index) => ($[stepColumn(index)] as CH.Expr).eq(1)) + const row = { + key: $.key as CH.Expr, + level: CH.windowFunnel(opts.windowSeconds * 1000)(ts, ...conditions), + } + if (plan.breakdownBy === undefined) return row + const dim = $.dim as CH.Expr + return { ...row, group: argMinIf(dim, ts, dim.neq("")) } + }) + .groupBy("key") +} + +/** `[countIf(level >= 1), …, countIf(level >= N)]` over a levels row set. */ +function stepCounts(stepCount: number): CH.Expr> { + const level = CH.dynamicColumn("level") + return CH.arrayOf(...Array.from({ length: stepCount }, (_, i) => CH.countIf(level.gte(i + 1)))) +} + +/** `arrayJoin([1, …, N])` — one output row per step. */ +function stepIndex(stepCount: number): CH.Expr { + return CH.arrayJoin(CH.arrayOf(...Array.from({ length: stepCount }, (_, i) => CH.lit(i + 1)))) +} + +// Public builders + +/** + * Per-step funnel counts: how many persons (per `keyBy`) reached at least step + * `n`, in order, within `windowSeconds` of their step-1 event. + * + * Always returns exactly `steps.length` rows, `step` 1..N ascending, so an + * empty range decodes as a row of zeros per step rather than as no rows. + * Conversion rates are the caller's division — the numbers here are the + * counts the UI shows next to them. + */ +export function productEventsFunnelQuery( + opts: ProductEventsFunnelOpts, +): CHQuery { + validate(opts) + const filters = opts.filters ?? {} + const first = opts.steps[0] + const plan: FunnelPlan = { + opts, + filters, + sessionStep: first?.kind === "session" ? first : undefined, + } + const n = opts.steps.length + + const totals = fromQuery(levelsQuery(plan), "levels").select(() => ({ counts: stepCounts(n) })) + + return fromQuery(totals, "totals") + .select(($) => ({ + step: stepIndex(n), + count: arrayElement($.counts, CH.dynamicColumn("step")), + })) + .orderBy(["step", "asc"]) + .format("JSON") +} + +/** + * The same funnel, split by one dimension: `{ group, step, count }` for the top + * `limit` groups by step-1 count. Every kept group emits all N steps. + * + * A person's group is the first non-empty value of the dimension across their + * rows in range (`argMinIf(dim, ts, dim != '')`) — for a session dimension that + * is their earliest session's referrer / campaign / country, for `host` the + * first host they were seen on, for `attribute:` the first event carrying + * it. Persons with no value at all land in the `''` group, which the caller + * may label or drop. + * + * Rows come back ordered by `group`, then `step`; the step-1 row of each group + * carries the rank the `limit` was applied on. + */ +export function productEventsFunnelBreakdownQuery( + opts: ProductEventsFunnelBreakdownOpts, +): CHQuery { + validate(opts) + const limit = opts.limit ?? 10 + if (!Number.isInteger(limit) || limit < 1 || limit > FUNNEL_BREAKDOWN_MAX_GROUPS) { + throw new ProductEventsFunnelError({ + reason: "InvalidLimit", + message: `breakdown limit must be an integer in 1..${FUNNEL_BREAKDOWN_MAX_GROUPS}, got ${String(limit)}`, + }) + } + const filters = opts.filters ?? {} + const first = opts.steps[0] + const plan: FunnelPlan = { + opts, + filters, + sessionStep: first?.kind === "session" ? first : undefined, + breakdownBy: opts.breakdownBy, + } + const n = opts.steps.length + + const perGroup = fromQuery(levelsQuery(plan), "levels") + .select(() => ({ + group: CH.dynamicColumn("group"), + counts: stepCounts(n), + entered: CH.countIf(CH.dynamicColumn("level").gte(1)), + })) + .groupBy("group") + .orderBy(["entered", "desc"], ["group", "asc"]) + .limit(limit) + + return fromQuery(perGroup, "groups") + .select(($) => ({ + group: $.group, + step: stepIndex(n), + count: arrayElement($.counts, CH.dynamicColumn("step")), + })) + .orderBy(["group", "asc"], ["step", "asc"]) + .format("JSON") +} + +/** + * Event names in range for the step picker: `{ eventName, kind, count, + * sessions, persons }`, most frequent first. + * + * `persons` is the unstitched `if(UserId != '', UserId, VisitorId)` — a cheap + * approximation that needs no `identity_links` join; the funnel itself does the + * stitching. Sidebar filters narrow by `SessionId`, so under an active filter + * server-side events (which carry no session) are not listed. + */ +export function productEventNamesQuery( + opts: ProductEventNamesOpts = {}, +): CHQuery { + const filters: ProductEventsFilters = { ...opts.filters, useProductEvents: true } + const limit = opts.limit ?? 100 + return from(ProductEvents) + .select(($) => ({ + eventName: $.EventName, + kind: $.Kind, + count: CH.count(), + sessions: CH.uniqIf($.SessionId, $.SessionId.neq("")), + persons: CH.uniq(CH.if_($.UserId.neq(""), $.UserId, $.VisitorId)), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(filters.host, (v: string) => $.Host.eq(v)), + needsSessionSemiJoin(filters) || filters.pagePath !== undefined + ? inSubquery( + $.SessionId, + from(SessionReplays) + .select(($s) => ({ sessionId: $s.SessionId })) + .where(($s) => replaysWhere($s, { ...filters, host: undefined })) + .groupBy("sessionId"), + ) + : undefined, + ]) + .groupBy("eventName", "kind") + .orderBy(["count", "desc"], ["eventName", "asc"]) + .limit(limit) + .format("JSON") +} + diff --git a/packages/query-engine/src/ch/queries/web-analytics.ts b/packages/query-engine/src/ch/queries/web-analytics.ts index 769a9b4f2..29a107743 100644 --- a/packages/query-engine/src/ch/queries/web-analytics.ts +++ b/packages/query-engine/src/ch/queries/web-analytics.ts @@ -11,12 +11,12 @@ import * as CH from "@maple-dev/clickhouse-builder/expr" import { param, from, inSubquery, unionAll, compileFnCall } from "@maple-dev/clickhouse-builder" import type { ColumnAccessor, CHQuery, CHUnionQuery } from "@maple-dev/clickhouse-builder" -import { SessionReplays, SessionEvents, WebEvents } from "../tables" +import { SessionReplays, SessionEvents, ProductEvents } from "../tables" import type { FacetOutput } from "./query-helpers" /** * The page-view discriminator: `session_events.Type` on the raw path, - * `web_events.Kind` on the rollup. Deliberately not `EventName = '$pageview'` — + * `product_events.Kind` on the rollup. Deliberately not `EventName = '$pageview'` — * `track()` takes a caller-supplied name with no reserved-prefix check, so a * customer calling `track('$pageview')` would inflate the count. `Kind` carries * the source `Type` through untouched, which is what makes the two paths @@ -64,7 +64,7 @@ export interface WebAnalyticsFilters { /** `new` keeps first-ever sessions for a visitor, `returning` the rest. */ readonly visitorType?: "new" | "returning" /** - * Read page views from the `web_events` rollup instead of `session_events`. + * Read page views from the `product_events` rollup instead of `session_events`. * * Purely a source swap — every predicate below has a one-to-one counterpart on * the rollup (`Type`→`Kind`, `domain(Url)`→`Host`, `path(Url)`→`PagePath`), @@ -73,9 +73,17 @@ export interface WebAnalyticsFilters { * property stays checkable: there is one definition of the filter semantics, * and only the table it resolves against changes. */ - readonly useWebEvents?: boolean + readonly useProductEvents?: boolean } +/** + * The same filter surface under the name the funnel queries use — the + * `/analytics` sidebar narrows page views and funnels identically, and + * `product-events.ts` narrows through {@link replaysWhere} exactly as + * the page-view queries do. + */ +export type ProductEventsFilters = WebAnalyticsFilters + /** Which `session_replays` dimensions a facet branch can exclude from its own WHERE. */ export type WebAnalyticsFacetKey = | "referrerHost" @@ -93,7 +101,7 @@ export type WebAnalyticsFacetKey = type ReplaysAccessor = ColumnAccessor type EventsAccessor = ColumnAccessor -type WebEventsAccessor = ColumnAccessor +type ProductEventsAccessor = ColumnAccessor /** * The page-view predicate over raw `session_events`. @@ -119,7 +127,7 @@ function navigationConditionsRaw( } /** - * The same predicate over the `web_events` rollup — the one-to-one counterpart + * The same predicate over the `product_events` rollup — the one-to-one counterpart * of {@link navigationConditionsRaw}, and the reason the two paths can be held * to byte-identical results. * @@ -129,7 +137,7 @@ function navigationConditionsRaw( * `PagePath` were parsed once at write time instead of once per scanned row. */ function navigationConditionsRollup( - $: WebEventsAccessor, + $: ProductEventsAccessor, filters: WebAnalyticsFilters, only?: "host" | "pagePath", ): Array { @@ -162,15 +170,15 @@ function navigationConditionsRollup( * * This subquery is inlined into **every one of the twelve breakdown branches** * whenever a page filter is active, which is what made a single top-pages click - * the most expensive interaction on the page. Pointing it at `web_events` is the - * main reason that table exists. + * the most expensive interaction on the page. Pointing it at `product_events` is + * the main reason that table exists. */ function navigationSessionsSubquery( filters: WebAnalyticsFilters, only?: "host" | "pagePath", ): CHQuery { - return filters.useWebEvents - ? from(WebEvents) + return filters.useProductEvents + ? from(ProductEvents) .select(($) => ({ sessionId: $.SessionId })) .where(($) => navigationConditionsRollup($, filters, only)) .groupBy("sessionId") @@ -187,7 +195,7 @@ function navigationSessionsSubquery( * doesn't collapse to the single selected value — the sidebar has to keep * offering the alternatives. */ -function replaysWhere( +export function replaysWhere( $: ReplaysAccessor, filters: WebAnalyticsFilters, exclude?: WebAnalyticsFacetKey, @@ -237,7 +245,7 @@ function replaysWhere( } /** True when any filter can only be evaluated against `session_replays`. */ -function needsSessionSemiJoin(filters: WebAnalyticsFilters): boolean { +export function needsSessionSemiJoin(filters: WebAnalyticsFilters): boolean { return Boolean( filters.referrerHost || filters.country || @@ -290,9 +298,9 @@ function navigationWhereRaw( return [...navigationConditionsRaw($, filters), replaysSemiJoin($.SessionId, filters)] } -/** WHERE conditions for the page-view queries over the `web_events` rollup. */ +/** WHERE conditions for the page-view queries over the `product_events` rollup. */ function navigationWhereRollup( - $: WebEventsAccessor, + $: ProductEventsAccessor, filters: WebAnalyticsFilters, ): Array { return [...navigationConditionsRollup($, filters), replaysSemiJoin($.SessionId, filters)] @@ -445,8 +453,8 @@ export function webAnalyticsPageviewsTimeseriesQuery( opts: WebAnalyticsPageviewsTimeseriesOpts = {}, ): CHQuery { const bucketSeconds = opts.bucketSeconds ?? 3600 - return opts.useWebEvents - ? from(WebEvents) + return opts.useProductEvents + ? from(ProductEvents) .select(($) => ({ bucket: CH.toStartOfInterval($.Timestamp, bucketSeconds), pageViews: CH.count(), @@ -497,8 +505,8 @@ export function webAnalyticsPagesQuery( opts: WebAnalyticsPagesOpts = {}, ): CHQuery { const limit = opts.limit ?? 100 - return opts.useWebEvents - ? from(WebEvents) + return opts.useProductEvents + ? from(ProductEvents) .select(($) => ({ host: $.Host, pagePath: $.PagePath, diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index a88867d80..de694d943 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -612,7 +612,7 @@ const webAnalyticsFilters = ( readonly utmCampaign?: string readonly visitorType?: "new" | "returning" }, - useWebEvents: boolean, + useProductEvents: boolean, ): CH.WebAnalyticsFilters => ({ host: payload.host, pagePath: payload.pagePath, @@ -626,17 +626,17 @@ const webAnalyticsFilters = ( utmMedium: payload.utmMedium, utmCampaign: payload.utmCampaign, visitorType: payload.visitorType, - useWebEvents, + useProductEvents, }) // Rollup/raw pairs share ids and cache keys because parity tests require identical results. -const webAnalyticsSummaryDef = (useWebEvents: boolean) => ({ +const webAnalyticsSummaryDef = (useProductEvents: boolean) => ({ id: "webAnalyticsSummary" as const, profile: "aggregation" as const, cache: timeRangeCache, compile: (payload: WebAnalyticsSummaryRequest, orgId: string) => - CH.compile(CH.webAnalyticsSummaryQuery(webAnalyticsFilters(payload, useWebEvents)), { + CH.compile(CH.webAnalyticsSummaryQuery(webAnalyticsFilters(payload, useProductEvents)), { orgId, startTime: payload.startTime, endTime: payload.endTime, @@ -646,14 +646,14 @@ const webAnalyticsSummaryDef = (useWebEvents: boolean) => ({ export const webAnalyticsSummary = defineQuery(webAnalyticsSummaryDef(true)) export const webAnalyticsSummaryRaw = defineQuery(webAnalyticsSummaryDef(false)) -const webAnalyticsTimeseriesDef = (useWebEvents: boolean) => ({ +const webAnalyticsTimeseriesDef = (useProductEvents: boolean) => ({ id: "webAnalyticsTimeseries" as const, profile: "aggregation" as const, cache: timeRangeCache, compile: (payload: WebAnalyticsTimeseriesRequest, orgId: string) => CH.compile( CH.webAnalyticsTimeseriesQuery({ - ...webAnalyticsFilters(payload, useWebEvents), + ...webAnalyticsFilters(payload, useProductEvents), bucketSeconds: payload.bucketSeconds, }), { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -663,14 +663,14 @@ const webAnalyticsTimeseriesDef = (useWebEvents: boolean) => ({ export const webAnalyticsTimeseries = defineQuery(webAnalyticsTimeseriesDef(true)) export const webAnalyticsTimeseriesRaw = defineQuery(webAnalyticsTimeseriesDef(false)) -const webAnalyticsPageviewsDef = (useWebEvents: boolean) => ({ +const webAnalyticsPageviewsDef = (useProductEvents: boolean) => ({ id: "webAnalyticsPageviews" as const, profile: "aggregation" as const, cache: timeRangeCache, compile: (payload: WebAnalyticsPageviewsRequest, orgId: string) => CH.compile( CH.webAnalyticsPageviewsTimeseriesQuery({ - ...webAnalyticsFilters(payload, useWebEvents), + ...webAnalyticsFilters(payload, useProductEvents), bucketSeconds: payload.bucketSeconds, }), { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -680,14 +680,14 @@ const webAnalyticsPageviewsDef = (useWebEvents: boolean) => ({ export const webAnalyticsPageviews = defineQuery(webAnalyticsPageviewsDef(true)) export const webAnalyticsPageviewsRaw = defineQuery(webAnalyticsPageviewsDef(false)) -const webAnalyticsPagesDef = (useWebEvents: boolean) => ({ +const webAnalyticsPagesDef = (useProductEvents: boolean) => ({ id: "webAnalyticsPages" as const, profile: "aggregation" as const, cache: timeRangeCache, compile: (payload: WebAnalyticsPagesRequest, orgId: string) => CH.compile( CH.webAnalyticsPagesQuery({ - ...webAnalyticsFilters(payload, useWebEvents), + ...webAnalyticsFilters(payload, useProductEvents), limit: payload.limit, }), { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -697,7 +697,7 @@ const webAnalyticsPagesDef = (useWebEvents: boolean) => ({ export const webAnalyticsPages = defineQuery(webAnalyticsPagesDef(true)) export const webAnalyticsPagesRaw = defineQuery(webAnalyticsPagesDef(false)) -const webAnalyticsBreakdownsDef = (useWebEvents: boolean) => ({ +const webAnalyticsBreakdownsDef = (useProductEvents: boolean) => ({ id: "webAnalyticsBreakdowns" as const, profile: "aggregation" as const, // Bound memory across the UNION fan-out. @@ -706,7 +706,7 @@ const webAnalyticsBreakdownsDef = (useWebEvents: boolean) => ({ compile: (payload: WebAnalyticsBreakdownsRequest, orgId: string) => CH.compileUnion( CH.webAnalyticsBreakdownsQuery({ - ...webAnalyticsFilters(payload, useWebEvents), + ...webAnalyticsFilters(payload, useProductEvents), limitPerDimension: payload.limitPerDimension, }), { orgId, startTime: payload.startTime, endTime: payload.endTime }, diff --git a/packages/query-engine/src/sql-catalog.test.ts b/packages/query-engine/src/sql-catalog.test.ts index 4f8cbc50c..daf17f2e5 100644 --- a/packages/query-engine/src/sql-catalog.test.ts +++ b/packages/query-engine/src/sql-catalog.test.ts @@ -31,6 +31,7 @@ import * as serviceQueries from "./ch/queries/services" import * as sessionEventQueries from "./ch/queries/session-events" import * as sessionReplayQueries from "./ch/queries/session-replays" import * as webAnalyticsQueries from "./ch/queries/web-analytics" +import * as productEventQueries from "./ch/queries/product-events" import * as topOperationQueries from "./ch/queries/top-operations" import * as traceQueries from "./ch/queries/traces" @@ -194,6 +195,7 @@ const QUERY_MODULES: Record> = { "session-replays": sessionReplayQueries, "top-operations": topOperationQueries, "web-analytics": webAnalyticsQueries, + "product-events": productEventQueries, traces: traceQueries, } satisfies Record> diff --git a/packages/query-engine/src/sql-catalog.ts b/packages/query-engine/src/sql-catalog.ts index 237f75cea..37399e5bb 100644 --- a/packages/query-engine/src/sql-catalog.ts +++ b/packages/query-engine/src/sql-catalog.ts @@ -948,14 +948,14 @@ export function routeCoverage(): ReadonlyMap Date: Mon, 17 Aug 2026 13:10:52 +0200 Subject: [PATCH 08/15] =?UTF-8?q?feat(web,mcp):=20funnels=20=E2=80=94=20/a?= =?UTF-8?q?nalytics=20Funnels=20tab,=20dashboard=20funnel=20widget=20block?= =?UTF-8?q?,=20query=5Ffunnel=20MCP=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal query-engine endpoints productEventsFunnel/-Breakdown/productEventNames; shared FunnelStep schema in @maple/query-model; additive display.funnel block on the existing funnel widget (mobile wire unchanged); query_funnel + list_product_events MCP tools. --- .../src/mcp/lib/dashboard-schema-doc.test.ts | 21 +- apps/api/src/mcp/lib/dashboard-schema-doc.ts | 58 ++- apps/api/src/mcp/resources/instructions.ts | 1 + .../mcp/tools/__tests__/query-funnel.test.ts | 192 +++++++++ .../api/src/mcp/tools/add-dashboard-widget.ts | 37 +- apps/api/src/mcp/tools/list-product-events.ts | 141 +++++++ apps/api/src/mcp/tools/query-funnel.ts | 296 +++++++++++++ apps/api/src/mcp/tools/registry.ts | 4 + .../src/routes/internal/query-engine.http.ts | 50 +++ apps/api/src/routes/query-helpers.ts | 42 +- .../dashboards/route-endpoint-plans.ts | 37 +- apps/web/src/api/warehouse/product-events.ts | 197 +++++++++ apps/web/src/api/warehouse/web-analytics.ts | 4 +- .../ai-elements/renderers/tool-renderer.tsx | 73 ++++ .../config/settings-fields.tsx | 108 +++++ .../dashboard-builder/data-source-registry.ts | 2 + .../src/components/dashboard-builder/types.ts | 1 + .../widgets/types/breakdown.tsx | 75 +++- .../widgets/widget-type-registry.ts | 7 + .../funnels/analytics-funnels-view.tsx | 396 ++++++++++++++++++ .../src/components/funnels/conversion.test.ts | 52 +++ apps/web/src/components/funnels/conversion.ts | 71 ++++ apps/web/src/components/funnels/definition.ts | 132 ++++++ .../src/components/funnels/funnel-results.tsx | 225 ++++++++++ .../funnels/funnel-step-builder.tsx | 379 +++++++++++++++++ .../query-builder/widget-builder-shared.ts | 21 +- .../widget-builder-utils.test.ts | 103 +++++ .../lib/query-builder/widget-builder-utils.ts | 9 + .../query-builder/widget-type-cycle.test.ts | 1 + .../services/atoms/warehouse-query-atoms.ts | 20 + apps/web/src/routes/analytics/index.tsx | 81 +++- packages/domain/src/http/query-engine.ts | 107 +++++ .../http/v2/dashboard-widget-parity.test.ts | 25 +- packages/domain/src/http/v2/dashboards.ts | 22 +- packages/domain/src/mcp-structured-types.ts | 41 ++ .../query-engine/src/observability/index.ts | 11 + .../src/observability/product-events.ts | 89 ++++ packages/query-engine/src/registry/index.ts | 1 + .../src/registry/product-events.ts | 83 ++++ packages/query-engine/src/registry/queries.ts | 1 + packages/query-model/src/funnel.ts | 83 ++++ packages/query-model/src/index.ts | 1 + .../widgets/src/dashboard/construct.test.ts | 29 +- packages/widgets/src/dashboard/construct.ts | 35 +- packages/widgets/src/dashboard/index.ts | 3 + .../widgets/src/dashboard/shared/display.ts | 14 +- 46 files changed, 3339 insertions(+), 42 deletions(-) create mode 100644 apps/api/src/mcp/tools/__tests__/query-funnel.test.ts create mode 100644 apps/api/src/mcp/tools/list-product-events.ts create mode 100644 apps/api/src/mcp/tools/query-funnel.ts create mode 100644 apps/web/src/api/warehouse/product-events.ts create mode 100644 apps/web/src/components/funnels/analytics-funnels-view.tsx create mode 100644 apps/web/src/components/funnels/conversion.test.ts create mode 100644 apps/web/src/components/funnels/conversion.ts create mode 100644 apps/web/src/components/funnels/definition.ts create mode 100644 apps/web/src/components/funnels/funnel-results.tsx create mode 100644 apps/web/src/components/funnels/funnel-step-builder.tsx create mode 100644 packages/query-engine/src/observability/product-events.ts create mode 100644 packages/query-engine/src/registry/product-events.ts create mode 100644 packages/query-model/src/funnel.ts diff --git a/apps/api/src/mcp/lib/dashboard-schema-doc.test.ts b/apps/api/src/mcp/lib/dashboard-schema-doc.test.ts index e9aa2e58a..955ee7f0d 100644 --- a/apps/api/src/mcp/lib/dashboard-schema-doc.test.ts +++ b/apps/api/src/mcp/lib/dashboard-schema-doc.test.ts @@ -47,8 +47,25 @@ describe("generated JSON examples decode", () => { // inferred that envelope correctly but reported it as a guess. const examples = jsonExamples(renderDashboardSchemaSection("data_sources")) const widgets = examples.filter((example) => Object.hasOwn(example as object, "dataSource")) - expect(widgets).toHaveLength(1) - expect(() => decodeWidget(widgets[0])).not.toThrow() + // The generic widget and the product-event funnel widget, whose data source + // is derived from `display.funnel` and so only makes sense shown whole. + expect(widgets).toHaveLength(2) + for (const widget of widgets) expect(() => decodeWidget(widget)).not.toThrow() + }) + + it("shows the product-event funnel definition on `display.funnel` with a derived route", () => { + const doc = renderDashboardSchemaSection("data_sources") + expect(doc).toContain("product_events_funnel") + const funnel = jsonExamples(doc).find( + (example) => + Object.hasOwn(example as object, "dataSource") && + (example as { visualization?: string }).visualization === "funnel", + ) as { + dataSource: { endpoint?: string; params?: { steps?: unknown[] } } + display: { funnel?: { steps?: unknown[] } } + } + expect(funnel.dataSource.endpoint).toBe("product_events_funnel") + expect(funnel.dataSource.params?.steps).toEqual(funnel.display.funnel?.steps) }) it("no example uses the retired v2 shape", () => { diff --git a/apps/api/src/mcp/lib/dashboard-schema-doc.ts b/apps/api/src/mcp/lib/dashboard-schema-doc.ts index e22d89630..cb392e94b 100644 --- a/apps/api/src/mcp/lib/dashboard-schema-doc.ts +++ b/apps/api/src/mcp/lib/dashboard-schema-doc.ts @@ -6,6 +6,7 @@ import { type WidgetTypeMeta, } from "@maple/domain/http" import { + makeProductEventsFunnelDataSource, makeQueryDataSource, makeRawSqlDataSource, makeStaticDataSource, @@ -140,6 +141,36 @@ const exampleBreakdownSource = () => ], }) +/** The steps a product-event funnel example runs; shared by the data-source and widget examples. */ +const exampleFunnelDefinition = () => ({ + steps: [ + { kind: "page" as const, pagePath: "/pricing" }, + { kind: "event" as const, eventName: "signup_completed" }, + { kind: "event" as const, eventName: "plan_started", attributeEquals: { plan: "pro" } }, + ], + keyBy: "person" as const, + windowSeconds: 7 * 24 * 3600, +}) + +/** + * A product-event funnel widget in full: the definition on `display.funnel` + * (what `add_dashboard_widget` reads) and the route data source it derives. + */ +const exampleFunnelWidget = () => { + const funnel = exampleFunnelDefinition() + return { + id: "w-signup-funnel", + visualization: "funnel", + dataSource: makeProductEventsFunnelDataSource(funnel), + display: { + title: "Signup funnel", + chartId: "query-builder-funnel", + funnel: { showStepPercent: true, ...funnel }, + }, + layout: { x: 0, y: 0, w: 6, h: 4 }, + } +} + /** * A complete persisted widget, not just its data source. * @@ -227,6 +258,31 @@ const dataSourcesSection = (): string => "", json(exampleBreakdownSource()), "", + '### Product-event funnels (`panel_type: "funnel"` + `display.funnel.steps`)', + "", + "A funnel widget has two modes. Without `display.funnel.steps` it draws a group-by breakdown", + "as descending stages (the shape above). With them it is a **conversion funnel over product", + "events** — page views, `track()` events and server-side events, stitched per person — and", + "the query set is not used at all. Set the definition on `display_json.funnel` and", + '`add_dashboard_widget` derives the data source (`kind: "route"`,', + '`endpoint: "product_events_funnel"`) for you; do not pass `data_source_json`.', + "", + '- `steps` — 1–10, in order. `{ kind: "event", eventName, attributeEquals? }`,', + ' `{ kind: "page", pagePath, host? }`, or — **step 1 only** —', + ' `{ kind: "session", dimension, value }` with `dimension` one of `referrerHost`,', + " `utmSource`, `utmMedium`, `utmCampaign`, `country`, `host`.", + "- `keyBy` — `person` (default; user id, else the visitor's linked user, else the visitor),", + " `visitor`, `user`, or `session`.", + "- `windowSeconds` — the whole chain must complete within this many seconds of step 1", + " (default 86400).", + "- `breakdownBy` — stored for parity with the /analytics Funnels view; the widget renders", + " the unsegmented funnel. Use `query_funnel` for a breakdown.", + "", + "Use `list_product_events` to see which event names exist, and `query_funnel` to try a", + "definition before pinning it to a board.", + "", + json(exampleFunnelWidget()), + "", "### A complete widget", "", "The sections above describe `add_dashboard_widget`'s parameters, which it assembles into a", @@ -371,7 +427,7 @@ const displaySection = (): string => "| `gauge` | gauge | `{ min, max }` — defaults to 0–100, which is wrong for a `percent` unit. |", "| `histogram` | histogram | `{ bucketCount, bucketWidth, logScaleY }`. |", "| `heatmap` | heatmap | `{ colorScale, scaleType }`. |", - "| `funnel` | funnel | `{ showStepPercent }`. |", + "| `funnel` | funnel | `{ showStepPercent, steps?, keyBy?, windowSeconds?, breakdownBy? }` — with `steps` it is a product-event funnel (see Data sources). |", "| `markdown` | markdown | `{ content }` — the note body. |", "| `sparkline` | stat | `{ enabled, dataSource? }`; embeds a full nested data source. |", "", diff --git a/apps/api/src/mcp/resources/instructions.ts b/apps/api/src/mcp/resources/instructions.ts index 26992faf7..55002477a 100644 --- a/apps/api/src/mcp/resources/instructions.ts +++ b/apps/api/src/mcp/resources/instructions.ts @@ -45,6 +45,7 @@ export const InstructionsResource = McpServer.resource({ - Trend analysis: query_data (timeseries or breakdown) - Service discovery: list_services -> diagnose_service - Alert management: list_alert_rules -> get_alert_rule -> create_alert_rule / update_alert_rule / delete_alert_rule -> list_alert_incidents +- Product analytics / conversion: list_product_events -> query_funnel (steps over page views, \`track()\` events and server events, stitched per person; \`breakdown_by\` a UTM/referrer dimension or an event attribute) -> add_dashboard_widget with \`panel_type: "funnel"\` and \`display_json.funnel.steps\` to pin it ## Dashboards diff --git a/apps/api/src/mcp/tools/__tests__/query-funnel.test.ts b/apps/api/src/mcp/tools/__tests__/query-funnel.test.ts new file mode 100644 index 000000000..cc73df629 --- /dev/null +++ b/apps/api/src/mcp/tools/__tests__/query-funnel.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "@effect/vitest" +import { Context, Effect, Option, Schema } from "effect" +import { WarehouseExecutor, productEventsFunnel } from "@maple/query-engine/observability" +import { CH } from "@maple/query-engine" +import type { McpToolRequirements } from "@/mcp/tools/runtime-requirements" +import type { McpToolRegistrar, McpToolResult } from "@/mcp/tools/types" +import { registerQueryFunnelTool } from "@/mcp/tools/query-funnel" +import { registerListProductEventsTool } from "@/mcp/tools/list-product-events" +import { mapleToolCatalog, toInputSchema } from "@/mcp/tools/registry" + +// Capture the handler the tool registers so its validation paths can be driven +// directly. Those return before any service is read, so an empty context is +// enough — the same trick `dispatcher.test.ts` uses. +type ToolInput = Record + +const captureTool = (register: (server: McpToolRegistrar) => void) => { + let captured: + | { + name: string + schema: Schema.Top + handler: (params: ToolInput) => Effect.Effect + } + | undefined + register({ + tool: (name, _description, schema, handler) => { + // SAFETY: the tests below pass inputs shaped by each tool's own Struct; + // the registrar erases the parameter type, so it is re-widened here. + captured = { name, schema, handler: (params) => handler(params as never) } + }, + }) + if (!captured) throw new Error("tool did not register") + return captured +} + +const run = (effect: Effect.Effect) => + Effect.runPromise( + (effect as Effect.Effect).pipe( + Effect.provide(Context.empty() as Context.Context), + ), + ) + +const text = (result: McpToolResult) => result.content.map((c) => ("text" in c ? c.text : "")).join("\n") + +describe("query_funnel / list_product_events registration", () => { + it("both tools are in the catalog with object input schemas", () => { + for (const name of ["query_funnel", "list_product_events"]) { + const definition = mapleToolCatalog.find((d) => d.name === name) + expect(definition, name).toBeDefined() + expect(toInputSchema(definition!.schema).type).toBe("object") + } + }) + + it("query_funnel requires steps_json and nothing else", () => { + const definition = mapleToolCatalog.find((d) => d.name === "query_funnel")! + expect(toInputSchema(definition.schema).required).toEqual(["steps_json"]) + }) +}) + +describe("query_funnel validation", () => { + const tool = captureTool(registerQueryFunnelTool) + + it("rejects malformed steps_json with the example", async () => { + const result = await run(tool.handler({ steps_json: "not json" })) + expect(result.isError).toBe(true) + expect(text(result)).toContain("Invalid steps_json") + expect(text(result)).toContain('"kind":"page"') + }) + + it("rejects a step of an unknown kind", async () => { + const result = await run( + tool.handler({ steps_json: JSON.stringify([{ kind: "click", target: "#buy" }]) }), + ) + expect(result.isError).toBe(true) + expect(text(result)).toContain("Invalid steps_json") + }) + + it("rejects an empty step list", async () => { + const result = await run(tool.handler({ steps_json: "[]" })) + expect(result.isError).toBe(true) + expect(text(result)).toContain("at least one step") + }) + + it("rejects a session step past step 1 before touching the warehouse", async () => { + const result = await run( + tool.handler({ + steps_json: JSON.stringify([ + { kind: "event", eventName: "signup_completed" }, + { kind: "session", dimension: "utmSource", value: "twitter" }, + ]), + }), + ) + expect(result.isError).toBe(true) + expect(text(result)).toContain("only valid as step 1") + }) + + it("rejects an unknown key_by and a non-positive window", async () => { + const steps = JSON.stringify([{ kind: "event", eventName: "x" }]) + const keyBy = await run(tool.handler({ steps_json: steps, key_by: "account" })) + expect(keyBy.isError).toBe(true) + expect(text(keyBy)).toContain("key_by must be one of") + + const window = await run(tool.handler({ steps_json: steps, window_seconds: 0 })) + expect(window.isError).toBe(true) + expect(text(window)).toContain("window_seconds") + }) + + it("rejects a breakdown_by outside the vocabulary but accepts attribute:", async () => { + const steps = JSON.stringify([{ kind: "event", eventName: "x" }]) + const bad = await run(tool.handler({ steps_json: steps, breakdown_by: "plan" })) + expect(bad.isError).toBe(true) + expect(text(bad)).toContain("breakdown_by must be one of") + // `attribute:plan` passes validation and proceeds to the tenant lookup, + // which the empty context cannot satisfy — that failure is the proof it + // got past the vocabulary check. + await expect( + run(tool.handler({ steps_json: steps, breakdown_by: "attribute:plan" })), + ).rejects.toThrow() + }) +}) + +describe("productEventsFunnel (observability helper)", () => { + const rows: ReadonlyArray<{ step: number; count: number }> = [ + { step: 1, count: 100 }, + { step: 2, count: 40 }, + ] + const compiledSql: string[] = [] + const executor = Context.make(WarehouseExecutor, { + orgId: "org_test", + query: () => Effect.succeed({ data: [] }), + compiledQuery: (compiled: { readonly sql: string }) => { + compiledSql.push(compiled.sql) + // SAFETY: the stub answers every compiled query with funnel rows; the + // only query these tests compile is the funnel, whose row type is `T`. + return Effect.succeed(rows as ReadonlyArray) + }, + compiledQueryFirst: () => Effect.succeed(Option.none()), + }) + + it.effect("compiles a definition and returns the executor's rows", () => + Effect.gen(function* () { + const result = yield* productEventsFunnel({ + startTime: "2026-08-10 00:00:00", + endTime: "2026-08-17 00:00:00", + steps: [ + { kind: "page", pagePath: "/pricing" }, + { kind: "event", eventName: "signup_completed" }, + ], + keyBy: "person", + windowSeconds: 86400, + }).pipe(Effect.provide(executor)) + expect(result).toEqual(rows) + expect(compiledSql.at(-1)).toContain("windowFunnel") + }), + ) + + it.effect("surfaces a builder rejection as ProductEventsFunnelError, not a defect", () => + Effect.gen(function* () { + const exit = yield* Effect.exit( + productEventsFunnel({ + startTime: "2026-08-10 00:00:00", + endTime: "2026-08-17 00:00:00", + steps: [ + { kind: "event", eventName: "signup_completed" }, + { kind: "session", dimension: "country", value: "DE" }, + ], + keyBy: "person", + windowSeconds: 86400, + }).pipe(Effect.provide(executor)), + ) + expect(exit._tag).toBe("Failure") + const failed = yield* Effect.flip( + productEventsFunnel({ + startTime: "2026-08-10 00:00:00", + endTime: "2026-08-17 00:00:00", + steps: [], + keyBy: "person", + windowSeconds: 86400, + }).pipe(Effect.provide(executor)), + ) + expect(failed).toBeInstanceOf(CH.ProductEventsFunnelError) + expect((failed as CH.ProductEventsFunnelError).reason).toBe("NoSteps") + }), + ) +}) + +describe("list_product_events registration shape", () => { + it("registers with only optional parameters", () => { + const tool = captureTool(registerListProductEventsTool) + expect(tool.name).toBe("list_product_events") + expect(toInputSchema(tool.schema).required ?? []).toEqual([]) + }) +}) diff --git a/apps/api/src/mcp/tools/add-dashboard-widget.ts b/apps/api/src/mcp/tools/add-dashboard-widget.ts index a67c282bb..115d89e6c 100644 --- a/apps/api/src/mcp/tools/add-dashboard-widget.ts +++ b/apps/api/src/mcp/tools/add-dashboard-widget.ts @@ -21,6 +21,7 @@ import { type DashboardWidget, } from "@/mcp/lib/dashboard-mutations" import { buildRawSqlDataSource, validateRawSqlMacro, withScalarReduction } from "@/mcp/lib/raw-sql-widget" +import { makeProductEventsFunnelDataSource } from "@maple/widgets/dashboard" import { PANEL_TYPE_LIST_MD, resolvePanelType } from "@/mcp/lib/panel-type" import { formatRenderIssues, validateWidgetRenderability } from "@/mcp/lib/validate-widget-renderability" import { @@ -77,7 +78,7 @@ export function registerAddDashboardWidgetTool(server: McpToolRegistrar) { "Bucket size in seconds for raw SQL timeseries. Only used when `sql` is set. If omitted the server auto-computes from the dashboard time range.", ), data_source_json: optionalStringParam( - "JSON string for the widget's dataSource: { endpoint, params?, transform? }. Required for the structured-query path; ignored when `sql` is set. Use get_dashboard on an existing widget to see the exact shape.", + "JSON string for the widget's dataSource: { endpoint, params?, transform? }. Required for the structured-query path; ignored when `sql` is set, and derived for you when `display_json.funnel.steps` defines a product-event funnel. Use get_dashboard on an existing widget to see the exact shape.", ), display_json: optionalStringParam( "JSON string for the widget's display config: { title?, unit?, thresholds?, chartId?, columns?, ... }. Required for the structured-query path; defaults to `{}` for the raw-SQL path. Use get_dashboard on an existing widget to see the exact shape.", @@ -106,17 +107,31 @@ export function registerAddDashboardWidgetTool(server: McpToolRegistrar) { time_range_json, }) { const useRawSql = typeof sql === "string" && sql.trim().length > 0 - if (!useRawSql && (!data_source_json || !display_json)) { - return validationError( - "add_dashboard_widget requires either `sql` (raw ClickHouse SQL path) or both `data_source_json` and `display_json` (structured-query path).", - '{ "sql": "SELECT count() FROM logs WHERE $__orgFilter AND $__timeFilter(Timestamp)" }', - ) - } const decodedDisplay: DashboardWidget["display"] = display_json ? yield* decodeDisplayJson(display_json, TOOL) : {} + // A product-event funnel is defined by `display_json.funnel.steps` alone: + // its data source is derived from that definition, so a caller need not + // (and should not) hand-assemble the route. + const funnelSteps = decodedDisplay.funnel?.steps + const funnelDefinition = + funnelSteps !== undefined && funnelSteps.length > 0 + ? { + steps: funnelSteps, + keyBy: decodedDisplay.funnel?.keyBy, + windowSeconds: decodedDisplay.funnel?.windowSeconds, + } + : undefined + + if (!useRawSql && funnelDefinition === undefined && (!data_source_json || !display_json)) { + return validationError( + "add_dashboard_widget requires either `sql` (raw ClickHouse SQL path), both `data_source_json` and `display_json` (structured-query path), or a `display_json.funnel.steps` definition (product-event funnel).", + '{ "sql": "SELECT count() FROM logs WHERE $__orgFilter AND $__timeFilter(Timestamp)" }', + ) + } + // One decision, one field. `panel_type` is preferred; `visualization` // stays accepted so agents and transcripts written against the old // surface keep working. @@ -154,6 +169,14 @@ export function registerAddDashboardWidgetTool(server: McpToolRegistrar) { displayType, granularitySeconds: granularity_seconds, }) + } else if (funnelDefinition !== undefined && !data_source_json) { + if (panel.visualization !== "funnel") { + return validationError( + `\`display_json.funnel.steps\` defines a product-event funnel, which only \`panel_type: "funnel"\` renders (got \`${panel.panelType}\`).`, + '{ "panel_type": "funnel", "display_json": "{\\"title\\":\\"Signup funnel\\",\\"funnel\\":{\\"steps\\":[{\\"kind\\":\\"page\\",\\"pagePath\\":\\"/pricing\\"},{\\"kind\\":\\"event\\",\\"eventName\\":\\"signup_completed\\"}]}}" }', + ) + } + dataSource = makeProductEventsFunnelDataSource(funnelDefinition) } else { dataSource = yield* decodeDataSourceJson(data_source_json!, TOOL) // A scalar tile reads `data[0].value`, so without a reduction it diff --git a/apps/api/src/mcp/tools/list-product-events.ts b/apps/api/src/mcp/tools/list-product-events.ts new file mode 100644 index 000000000..8dc952b1d --- /dev/null +++ b/apps/api/src/mcp/tools/list-product-events.ts @@ -0,0 +1,141 @@ +import { optionalNumberParam, optionalStringParam, type McpToolRegistrar } from "./types" +import { warehouseToMcpHandlers } from "@/mcp/lib/map-warehouse-error" +import { withTenantExecutor, CurrentMcpTenant } from "@/mcp/lib/query-warehouse" +import { resolveTimeRange, rangeExceededResult, MCP_DISCOVERY_MAX_HOURS } from "@/mcp/lib/time" +import { clampLimit } from "@/mcp/lib/limits" +import { formatTable, formatNumber, truncate } from "@/mcp/lib/format" +import { formatNextSteps } from "@/mcp/lib/next-steps" +import { createDualContent } from "@/mcp/lib/structured-output" +import { Effect, Schema } from "effect" +import type { ListProductEventsData } from "@maple/domain" +import { productEventNames } from "@maple/query-engine/observability" + +const TOOL = "list_product_events" +const DEFAULT_RANGE_HOURS = 7 * 24 + +export function registerListProductEventsTool(server: McpToolRegistrar) { + server.tool( + TOOL, + "List the product event names an org has recorded — browser `track()` events, server-side events and page views — with how often each fired and how many sessions and persons it reached. Use it to discover the step names for `query_funnel` (an event step needs an exact `eventName`). `kind` tells them apart: `custom` is a `track()`/server event, `navigation` is a page view (`$pageview`), `screen` a mobile screen. Filters (`host`, `page_path`, `referrer_host`, `country`, `utm_*`) narrow to events from matching sessions.", + Schema.Struct({ + start_time: optionalStringParam( + "Start of time range (YYYY-MM-DD HH:mm:ss). Default: last 7 days.", + ), + end_time: optionalStringParam("End of time range (YYYY-MM-DD HH:mm:ss)."), + kind: optionalStringParam( + "Only events of this kind: `custom`, `navigation` or `screen`. Default: all.", + ), + search: optionalStringParam("Case-insensitive substring match on the event name."), + host: optionalStringParam("Only events from sessions on this site host."), + page_path: optionalStringParam("Only events from sessions that viewed this page path."), + referrer_host: optionalStringParam("Only events from sessions referred by this host."), + country: optionalStringParam("Only events from sessions in this country (ISO code)."), + utm_source: optionalStringParam("Only events from sessions carrying this utm_source."), + utm_medium: optionalStringParam("Only events from sessions carrying this utm_medium."), + utm_campaign: optionalStringParam("Only events from sessions carrying this utm_campaign."), + limit: optionalNumberParam("Max event names to return (default 50, max 200)."), + }), + Effect.fn("McpTool.listProductEvents")(function* (params) { + const range = resolveTimeRange(params.start_time, params.end_time, { + defaultHours: DEFAULT_RANGE_HOURS, + maxHours: MCP_DISCOVERY_MAX_HOURS, + }) + const { st, et } = range + if (range.exceeded) return rangeExceededResult(range, TOOL) + const limit = clampLimit(params.limit, { defaultValue: 50, max: 200 }) + + const tenant = yield* CurrentMcpTenant + yield* Effect.annotateCurrentSpan({ orgId: tenant.orgId, kind: params.kind ?? "any", limit }) + + // The query ranks every name; kind and name filters apply here so the + // limit still means "names shown". Fetch the full cap when narrowing. + const narrowing = params.kind !== undefined || params.search !== undefined + const rows = yield* withTenantExecutor( + productEventNames({ + startTime: st, + endTime: et, + limit: narrowing ? 200 : limit, + filters: { + host: params.host ?? undefined, + pagePath: params.page_path ?? undefined, + referrerHost: params.referrer_host ?? undefined, + country: params.country ?? undefined, + utmSource: params.utm_source ?? undefined, + utmMedium: params.utm_medium ?? undefined, + utmCampaign: params.utm_campaign ?? undefined, + }, + }), + ).pipe(Effect.catchTags(warehouseToMcpHandlers(TOOL))) + + const search = params.search?.toLowerCase() + const events = rows + .filter((row) => params.kind === undefined || row.kind === params.kind) + .filter((row) => search === undefined || row.eventName.toLowerCase().includes(search)) + .slice(0, limit) + .map((row) => ({ + eventName: row.eventName, + kind: row.kind, + count: Number(row.count) || 0, + sessions: Number(row.sessions) || 0, + persons: Number(row.persons) || 0, + })) + + yield* Effect.annotateCurrentSpan("result.rowCount", events.length) + + if (events.length === 0) { + const custom = params.kind === undefined || params.kind === "custom" + return { + content: [ + { + type: "text" as const, + text: [ + `No product events matched (${st} — ${et}).`, + custom + ? 'To record product events call `maple.track("signup_completed", { plan: "pro" })` from the browser SDK, or `MapleEvents.track()` server-side; each name then shows up here and can be a funnel step.' + : "", + ] + .filter(Boolean) + .join("\n"), + }, + ], + } + } + + const lines: string[] = [ + `## Product events (${events.length}${narrowing ? " matching" : ""})`, + `Time range: ${st} — ${et}`, + "", + formatTable( + ["Event", "Kind", "Count", "Sessions", "Persons"], + events.map((event) => [ + truncate(event.eventName, 60), + event.kind, + formatNumber(event.count), + formatNumber(event.sessions), + formatNumber(event.persons), + ]), + ), + ] + + const customEvents = events.filter((event) => event.kind === "custom") + if (customEvents.length === 0) { + lines.push( + "", + "Only page views so far — no `track()` events. Page steps still work in `query_funnel`; custom events come from `maple.track(name, props)` in the browser SDK or `MapleEvents.track()` server-side.", + ) + } + + const suggested = customEvents.slice(0, 2).map((event) => event.eventName) + lines.push( + formatNextSteps([ + suggested.length > 0 + ? `\`query_funnel steps_json='${JSON.stringify(suggested.map((eventName) => ({ kind: "event", eventName })))}'\` — measure conversion between them` + : '`query_funnel steps_json=\'[{"kind":"page","pagePath":"/"},{"kind":"page","pagePath":"/pricing"}]\'\` — a page-to-page funnel', + ]), + ) + + const data: ListProductEventsData = { timeRange: { start: st, end: et }, events } + return { content: createDualContent(lines.join("\n"), { tool: TOOL, data }) } + }), + ) +} diff --git a/apps/api/src/mcp/tools/query-funnel.ts b/apps/api/src/mcp/tools/query-funnel.ts new file mode 100644 index 000000000..98a749cb3 --- /dev/null +++ b/apps/api/src/mcp/tools/query-funnel.ts @@ -0,0 +1,296 @@ +import { + optionalNumberParam, + optionalStringParam, + requiredStringParam, + validationError, + type McpToolRegistrar, +} from "./types" +import { warehouseToMcpHandlers } from "@/mcp/lib/map-warehouse-error" +import { withTenantExecutor, CurrentMcpTenant } from "@/mcp/lib/query-warehouse" +import { resolveTimeRange, rangeExceededResult, MCP_DISCOVERY_MAX_HOURS } from "@/mcp/lib/time" +import { clampLimit } from "@/mcp/lib/limits" +import { formatTable, formatNumber, formatPercent, truncate } from "@/mcp/lib/format" +import { formatNextSteps } from "@/mcp/lib/next-steps" +import { createDualContent } from "@/mcp/lib/structured-output" +import { Effect, Result, Schema } from "effect" +import { + FUNNEL_MAX_STEPS, + FunnelBreakdownBy, + FunnelKeyBy, + FunnelStep, + funnelStepLabel, + type FunnelKeyBy as FunnelKeyByType, + type FunnelStep as FunnelStepType, +} from "@maple/query-model" +import type { QueryFunnelData } from "@maple/domain" +import { productEventsFunnel, productEventsFunnelBreakdown } from "@maple/query-engine/observability" + +const TOOL = "query_funnel" + +const StepsFromJson = Schema.fromJsonString(Schema.Array(FunnelStep)) +const decodeSteps = Schema.decodeEffect(StepsFromJson) +const decodeBreakdownBy = Schema.decodeUnknownOption(FunnelBreakdownBy) +const decodeKeyBy = Schema.decodeUnknownOption(FunnelKeyBy) + +const DEFAULT_WINDOW_SECONDS = 24 * 3600 +const DEFAULT_RANGE_HOURS = 7 * 24 +const BREAKDOWN_MAX_GROUPS = 20 + +const STEPS_EXAMPLE = + '[{"kind":"page","pagePath":"/pricing"},{"kind":"event","eventName":"signup_completed"},{"kind":"event","eventName":"plan_started","attributeEquals":{"plan":"pro"}}]' + +const KEY_BY_NOUN = { + person: "persons", + visitor: "visitors", + user: "users", + session: "sessions", +} satisfies Record + +/** Share of `denominator`, or null when there is nothing to divide by. */ +const ratio = (numerator: number, denominator: number): number | null => + denominator > 0 ? numerator / denominator : null + +const fmtPct = (fraction: number | null): string => (fraction === null ? "—" : formatPercent(fraction)) + +export function registerQueryFunnelTool(server: McpToolRegistrar) { + server.tool( + TOOL, + 'Run a conversion funnel over product events — page views, browser `track()` events and server-side events, stitched per person. Give 1–10 ordered steps as `steps_json`; each step is `{kind:"event", eventName, attributeEquals?}`, `{kind:"page", pagePath, host?}`, or (step 1 only) `{kind:"session", dimension, value}` for how the session was acquired (`dimension`: referrerHost | utmSource | utmMedium | utmCampaign | country | host). Returns per-step counts, share of step 1, step-to-step conversion and drop-off; optionally broken down by an acquisition dimension or an event attribute (`breakdown_by`: one of the session dimensions, or `attribute:`). Call `list_product_events` first to see which event names exist. Filters (`host`, `page_path`, `referrer_host`, `country`, `utm_*`, `device_type`, `browser`) narrow the population to persons with a matching session.', + Schema.Struct({ + steps_json: requiredStringParam( + `JSON array of 1–${FUNNEL_MAX_STEPS} funnel steps, in order. Example: ${STEPS_EXAMPLE}`, + ), + key_by: optionalStringParam( + "What to count: `person` (default — user id when known, else the visitor's linked user, else the visitor), `visitor`, `user`, or `session` (per-session funnel; server events take no part).", + ), + window_seconds: optionalNumberParam( + "The whole chain must complete within this many seconds of the step-1 event. Default 86400 (24h).", + ), + breakdown_by: optionalStringParam( + "Group persons by `referrerHost`, `utmSource`, `utmMedium`, `utmCampaign`, `country`, `host`, or `attribute:` (an attribute on their events). Top groups by step-1 count.", + ), + breakdown_limit: optionalNumberParam( + `Groups to keep when breaking down (default 10, max ${BREAKDOWN_MAX_GROUPS}).`, + ), + start_time: optionalStringParam( + "Start of time range (YYYY-MM-DD HH:mm:ss). Default: last 7 days.", + ), + end_time: optionalStringParam("End of time range (YYYY-MM-DD HH:mm:ss)."), + host: optionalStringParam("Only persons with a session on this site host."), + page_path: optionalStringParam("Only persons with a session that viewed this page path."), + referrer_host: optionalStringParam("Only persons whose session was referred by this host."), + country: optionalStringParam("Only persons with a session from this country (ISO code)."), + utm_source: optionalStringParam("Only persons with a session carrying this utm_source."), + utm_medium: optionalStringParam("Only persons with a session carrying this utm_medium."), + utm_campaign: optionalStringParam("Only persons with a session carrying this utm_campaign."), + device_type: optionalStringParam( + "Only persons with a session on this device type (desktop, mobile, tablet).", + ), + browser: optionalStringParam("Only persons with a session in this browser (e.g. Chrome)."), + }), + Effect.fn("McpTool.queryFunnel")(function* (params) { + const range = resolveTimeRange(params.start_time, params.end_time, { + defaultHours: DEFAULT_RANGE_HOURS, + maxHours: MCP_DISCOVERY_MAX_HOURS, + }) + const { st, et } = range + if (range.exceeded) return rangeExceededResult(range, TOOL) + + const stepsResult = yield* Effect.result(decodeSteps(params.steps_json)) + if (Result.isFailure(stepsResult)) { + return validationError(`Invalid steps_json: ${String(stepsResult.failure)}`, STEPS_EXAMPLE) + } + const steps: ReadonlyArray = stepsResult.success + if (steps.length === 0) + return validationError("steps_json must contain at least one step.", STEPS_EXAMPLE) + if (steps.length > FUNNEL_MAX_STEPS) { + return validationError(`A funnel has at most ${FUNNEL_MAX_STEPS} steps, got ${steps.length}.`) + } + const lateSession = steps.findIndex((step, index) => index > 0 && step.kind === "session") + if (lateSession !== -1) { + return validationError( + `A session step is only valid as step 1, found one at step ${lateSession + 1}.`, + '[{"kind":"session","dimension":"utmSource","value":"twitter"},{"kind":"event","eventName":"signup_completed"}]', + ) + } + + const keyByOption = params.key_by === undefined ? undefined : decodeKeyBy(params.key_by) + if (keyByOption !== undefined && keyByOption._tag === "None") { + return validationError( + `key_by must be one of ${FunnelKeyBy.literals.join(", ")}; got "${params.key_by}".`, + ) + } + const keyBy = keyByOption === undefined ? "person" : keyByOption.value + + const windowSeconds = params.window_seconds ?? DEFAULT_WINDOW_SECONDS + if (!Number.isFinite(windowSeconds) || windowSeconds <= 0) { + return validationError( + `window_seconds must be a positive number; got ${String(params.window_seconds)}.`, + ) + } + + const breakdownOption = + params.breakdown_by === undefined ? undefined : decodeBreakdownBy(params.breakdown_by) + if (breakdownOption !== undefined && breakdownOption._tag === "None") { + return validationError( + `breakdown_by must be one of referrerHost, utmSource, utmMedium, utmCampaign, country, host, or attribute:; got "${params.breakdown_by}".`, + '{ "breakdown_by": "attribute:plan" }', + ) + } + const breakdownBy = breakdownOption?.value + + const filters = { + host: params.host ?? undefined, + pagePath: params.page_path ?? undefined, + referrerHost: params.referrer_host ?? undefined, + country: params.country ?? undefined, + utmSource: params.utm_source ?? undefined, + utmMedium: params.utm_medium ?? undefined, + utmCampaign: params.utm_campaign ?? undefined, + deviceType: params.device_type ?? undefined, + browserName: params.browser ?? undefined, + } + + const tenant = yield* CurrentMcpTenant + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + steps: steps.length, + keyBy, + windowSeconds, + breakdownBy: breakdownBy ?? "none", + }) + + const definition = { steps, keyBy, windowSeconds, filters, startTime: st, endTime: et } + + // The builder's own validation is the last word (it also catches what the + // checks above did not think of); its rejection is a caller error, not a + // tool failure. + const outcome = yield* withTenantExecutor(productEventsFunnel(definition)).pipe( + Effect.catchTags(warehouseToMcpHandlers(TOOL)), + Effect.catchTag("@maple/query-engine/ProductEventsFunnelError", (error) => + Effect.succeed({ invalid: error.message }), + ), + ) + if ("invalid" in outcome) return validationError(outcome.invalid, STEPS_EXAMPLE) + const counts = new Map(outcome.map((row) => [Number(row.step), Number(row.count) || 0])) + + const first = counts.get(1) ?? 0 + const stepData = steps.map((step, index) => { + const count = counts.get(index + 1) ?? 0 + const previous = index === 0 ? null : (counts.get(index) ?? 0) + return { + step: index + 1, + label: funnelStepLabel(step), + count, + ofFirst: index === 0 ? (first > 0 ? 1 : 0) : (ratio(count, first) ?? 0), + ofPrevious: previous === null ? null : ratio(count, previous), + dropOff: previous === null ? 0 : Math.max(0, previous - count), + } + }) + const conversion = steps.length < 2 ? null : ratio(stepData[stepData.length - 1]!.count, first) + const noun = KEY_BY_NOUN[keyBy] + + const lines: string[] = [ + `## Funnel (${steps.length} step${steps.length === 1 ? "" : "s"}, by ${keyBy}, within ${windowSeconds}s)`, + `Time range: ${st} — ${et}`, + "", + ] + if (first === 0) { + lines.push(`Nobody matched step 1 (${funnelStepLabel(steps[0]!)}) in this window.`) + } else { + lines.push( + formatTable( + [ + "#", + "Step", + noun[0]!.toUpperCase() + noun.slice(1), + "Of first", + "Of previous", + "Drop-off", + ], + stepData.map((stat) => [ + String(stat.step), + truncate(stat.label, 60), + formatNumber(stat.count), + fmtPct(stat.ofFirst), + fmtPct(stat.ofPrevious), + stat.step === 1 ? "—" : `-${formatNumber(stat.dropOff)}`, + ]), + ), + "", + conversion === null + ? "Add a second step to measure conversion." + : `**Conversion: ${formatPercent(conversion)}** (${formatNumber(stepData[stepData.length - 1]!.count)} of ${formatNumber(first)} ${noun}).`, + ) + } + + let breakdown: QueryFunnelData["breakdown"] + if (breakdownBy !== undefined && first > 0) { + const limit = clampLimit(params.breakdown_limit, { + defaultValue: 10, + max: BREAKDOWN_MAX_GROUPS, + }) + const groupRows = yield* withTenantExecutor( + productEventsFunnelBreakdown({ ...definition, breakdownBy, limit }), + ).pipe( + Effect.catchTags(warehouseToMcpHandlers(TOOL)), + // The definition already ran once above, so a builder rejection here + // cannot happen; keep the channel typed rather than dying on it. + Effect.catchTag("@maple/query-engine/ProductEventsFunnelError", () => Effect.succeed([])), + ) + const byGroup = new Map() + for (const row of groupRows) { + const group = String(row.group) + let arr = byGroup.get(group) + if (!arr) { + arr = new Array(steps.length).fill(0) + byGroup.set(group, arr) + } + const index = Number(row.step) - 1 + if (index >= 0 && index < steps.length) arr[index] = Number(row.count) || 0 + } + const groups = [...byGroup.entries()].map(([group, groupCounts]) => ({ + group, + counts: groupCounts, + conversion: + steps.length < 2 + ? null + : ratio(groupCounts[groupCounts.length - 1] ?? 0, groupCounts[0] ?? 0), + })) + breakdown = { by: breakdownBy, groups } + + lines.push("", `### By ${breakdownBy} (top ${groups.length} by step 1)`, "") + lines.push( + formatTable( + [breakdownBy, ...steps.map((_, index) => `Step ${index + 1}`), "Conv."], + groups.map((group) => [ + group.group === "" ? "(none)" : truncate(group.group, 40), + ...group.counts.map((count) => formatNumber(count)), + fmtPct(group.conversion), + ]), + ), + ) + } + + lines.push( + formatNextSteps([ + "`list_product_events` — see which event names exist before adding a step", + '`add_dashboard_widget panel_type="funnel"` with `display_json.funnel.steps` — pin this funnel to a board', + breakdownBy === undefined + ? '`query_funnel breakdown_by="utmSource"` — see where the converters came from' + : `\`search_sessions\` — read the sessions behind a group`, + ]), + ) + + const data: QueryFunnelData = { + timeRange: { start: st, end: et }, + keyBy, + windowSeconds, + steps: stepData, + conversion, + ...(breakdown !== undefined ? { breakdown } : undefined), + } + return { content: createDualContent(lines.join("\n"), { tool: TOOL, data }) } + }), + ) +} diff --git a/apps/api/src/mcp/tools/registry.ts b/apps/api/src/mcp/tools/registry.ts index 6847846f1..72c3379b6 100644 --- a/apps/api/src/mcp/tools/registry.ts +++ b/apps/api/src/mcp/tools/registry.ts @@ -49,6 +49,8 @@ import { registerMineLogPatternsTool } from "./mine-log-patterns" import { registerSearchLogsTool } from "./search-logs" import { registerSearchTracesTool } from "./search-traces" import { registerSearchSessionsTool } from "./search-sessions" +import { registerQueryFunnelTool } from "./query-funnel" +import { registerListProductEventsTool } from "./list-product-events" import { registerGetSessionTranscriptTool } from "./get-session-transcript" import { registerGetSessionTracesTool } from "./get-session-traces" import { registerServiceMapTool } from "./service-map" @@ -186,6 +188,8 @@ const collectMapleToolDefinitions = (): ReadonlyArray => { registerMineLogPatternsTool(registrar) registerSearchTracesTool(registrar) registerSearchSessionsTool(registrar) + registerQueryFunnelTool(registrar) + registerListProductEventsTool(registrar) registerGetSessionTranscriptTool(registrar) registerGetSessionTracesTool(registrar) registerDiagnoseServiceTool(registrar) diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 9f34cadcd..065b8206c 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -61,6 +61,9 @@ import { WebAnalyticsPageviewsResponse, WebAnalyticsPagesResponse, WebAnalyticsBreakdownsResponse, + ProductEventsFunnelResponse, + ProductEventsFunnelBreakdownResponse, + ProductEventNamesResponse, CommitSha, FingerprintHash, ServiceName, @@ -88,9 +91,11 @@ import { partitionWindowAround, podMetricSpec, toCloudflareFilters, + validateFunnelDefinition, workloadMetricSpec, } from "@/routes/query-helpers" import { Queries } from "@/routes/queries" +import { productEventsFunnelOpts } from "@maple/query-engine/registry" import { makeQueryRunners } from "@/routes/query-runner" import { runQueryEngineBatch } from "@/routes/query-engine-batch" import type { ExecutionTenant, WarehouseExecutionError } from "@maple/query-engine/execution" @@ -1733,6 +1738,51 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query return new WebAnalyticsBreakdownsResponse({ data: buckets }) }), ) + // Funnels have no raw-`session_events` fallback: server and mobile + // events exist only in `product_events`, so a cluster without the table + // surfaces the missing-table error instead of a silently smaller funnel. + .handle("productEventsFunnel", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* validateFunnelDefinition(productEventsFunnelOpts(payload)) + const rows = yield* runQuery(Queries.productEventsFunnel, tenant, payload) + return new ProductEventsFunnelResponse({ + data: rows.map((row) => ({ + step: Number(row.step) || 0, + count: Number(row.count) || 0, + })), + }) + }), + ) + .handle("productEventsFunnelBreakdown", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* validateFunnelDefinition(productEventsFunnelOpts(payload)) + const rows = yield* runQuery(Queries.productEventsFunnelBreakdown, tenant, payload) + return new ProductEventsFunnelBreakdownResponse({ + data: rows.map((row) => ({ + group: String(row.group), + step: Number(row.step) || 0, + count: Number(row.count) || 0, + })), + }) + }), + ) + .handle("productEventNames", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventNames, tenant, payload) + return new ProductEventNamesResponse({ + data: rows.map((row) => ({ + eventName: String(row.eventName), + kind: String(row.kind), + count: Number(row.count) || 0, + sessions: Number(row.sessions) || 0, + persons: Number(row.persons) || 0, + })), + }) + }), + ) .handle("executeRawSql", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/routes/query-helpers.ts b/apps/api/src/routes/query-helpers.ts index 409075a40..796831187 100644 --- a/apps/api/src/routes/query-helpers.ts +++ b/apps/api/src/routes/query-helpers.ts @@ -1,11 +1,13 @@ import * as Integrations from "@maple/query-engine-integrations" -import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" -import type { - HostInfraTimeseriesRequest, - NodeInfraTimeseriesRequest, - PodInfraTimeseriesRequest, - WorkloadInfraTimeseriesRequest, +import { CH, formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" +import { + QueryEngineValidationError, + type HostInfraTimeseriesRequest, + type NodeInfraTimeseriesRequest, + type PodInfraTimeseriesRequest, + type WorkloadInfraTimeseriesRequest, } from "@maple/domain/http" +import { Effect, Schema } from "effect" /** * Helpers shared between the query-engine handlers and the app-side query @@ -159,3 +161,31 @@ export const hostMetricSpec = (metric: HostInfraTimeseriesRequest["metric"]) => } } } + +const isProductEventsFunnelError = Schema.is(CH.ProductEventsFunnelError) + +/** + * A funnel definition the query builder cannot compile is a caller error, not a + * warehouse one. The builders validate synchronously and throw + * `ProductEventsFunnelError`; the registry's `compile` would turn that into a + * defect (a 500 with no remediation), so the definition is checked here first + * and the reason lands in the 400 envelope. Anything else thrown is a genuine + * defect and stays one. Shared by the internal endpoint and the share API's + * `product_events_funnel` route plan. + */ +export const validateFunnelDefinition = ( + opts: CH.ProductEventsFunnelOpts, +): Effect.Effect => + Effect.suspend(() => { + try { + CH.productEventsFunnelQuery(opts) + return Effect.void + } catch (error) { + if (isProductEventsFunnelError(error)) { + return Effect.fail( + new QueryEngineValidationError({ message: error.message, details: [error.reason] }), + ) + } + return Effect.die(error) + } + }) diff --git a/apps/api/src/services/dashboards/route-endpoint-plans.ts b/apps/api/src/services/dashboards/route-endpoint-plans.ts index 9549430c6..81b4ebebc 100644 --- a/apps/api/src/services/dashboards/route-endpoint-plans.ts +++ b/apps/api/src/services/dashboards/route-endpoint-plans.ts @@ -31,9 +31,12 @@ import { ErrorsByTypeRequest, ErrorsSummaryRequest, ListLogsRequest, + ProductEventsFunnelRequest, ServiceOverviewRequest, ServiceUsageRequest, } from "@maple/domain/http" +import { funnelStepLabel } from "@maple/query-model" +import { PRODUCT_EVENTS_FUNNEL_ENDPOINT } from "@maple/widgets/dashboard" import { Effect, Schema } from "effect" import { coerceErrorsByTypeRows, @@ -44,7 +47,8 @@ import { serviceUsagePreviousTotals, windowDurationSeconds, } from "@maple/query-engine" -import { Queries, type QueryDefinition } from "@maple/query-engine/registry" +import { Queries, productEventsFunnelOpts, type QueryDefinition } from "@maple/query-engine/registry" +import { validateFunnelDefinition } from "@/routes/query-helpers" import { makeQueryRunners } from "@/routes/query-runner" import type { QueryEngineServiceApi } from "@/services/warehouse/QueryEngineService" import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" @@ -122,6 +126,8 @@ const asRows = (rows: ReadonlyArray): ReadonlyArray> +const decodeProductEventsFunnel = Schema.decodeUnknownEffect(ProductEventsFunnelRequest) + export const ROUTE_ENDPOINT_PLANS: RouteEndpointPlanRegistry = { errors_by_type: readModelPlan(ErrorsByTypeRequest, Queries.errorsByType, (rows) => ({ data: coerceErrorsByTypeRows(asRows(rows)), @@ -156,6 +162,35 @@ export const ROUTE_ENDPOINT_PLANS: RouteEndpointPlanRegistry = { const cursor = logs.length === limit && logs.length > 0 ? logs[logs.length - 1].timestamp : null return { data: logs, meta: { limit, cursor } } }), + // The product-event funnel widget. The stored params carry the definition + // (`steps`, optional `keyBy`/`windowSeconds`); the defaults and the + // `{ name, value }` row shape match the browser's `getProductEventsFunnelWidget` + // exactly, so a shared funnel tile draws the same bars as the signed-in one. + [PRODUCT_EVENTS_FUNNEL_ENDPOINT]: { + run: (params, context) => + Effect.gen(function* () { + const payload = yield* decodeProductEventsFunnel({ + keyBy: "person", + windowSeconds: 24 * 3600, + ...params, + startTime: context.window.startTime, + endTime: context.window.endTime, + }) + yield* validateFunnelDefinition(productEventsFunnelOpts(payload)) + const { runQuery } = makeQueryRunners({ + warehouse: context.warehouse, + queryEngine: context.queryEngine, + }) + const rows = yield* runQuery(Queries.productEventsFunnel, context.tenant, payload) + const countByStep = new Map(rows.map((row) => [Number(row.step), Number(row.count) || 0])) + return { + data: payload.steps.map((step, index) => ({ + name: funnelStepLabel(step), + value: countByStep.get(index + 1) ?? 0, + })), + } + }), + }, } /** Endpoints a shared dashboard can render. Used by the dialog's warning list. */ diff --git a/apps/web/src/api/warehouse/product-events.ts b/apps/web/src/api/warehouse/product-events.ts new file mode 100644 index 000000000..ee1c3827a --- /dev/null +++ b/apps/web/src/api/warehouse/product-events.ts @@ -0,0 +1,197 @@ +// Product-event funnels: the step-based conversion queries over +// `product_events`. Shares the web-analytics filter surface so the /analytics +// sidebar narrows a funnel exactly the way it narrows the page-view panels. +// See packages/query-engine/src/ch/queries/product-events.ts for the semantics +// of `keyBy`, the session step, and the breakdown grouping. + +import { Effect, Schema } from "effect" +import { + FunnelBreakdownBy, + FunnelKeyBy, + FunnelStep, + ProductEventNamesRequest, + ProductEventsFunnelBreakdownRequest, + ProductEventsFunnelRequest, +} from "@maple/domain/http" +import { funnelStepLabel } from "@maple/query-model" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" +import { decodeInput, runWarehouseQuery } from "@/api/warehouse/effect-utils" +import { TimeWindowFields, WebAnalyticsFilterFields } from "@/api/warehouse/web-analytics" + +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) + +const ProductEventsFunnelInputSchema = Schema.Struct({ + ...TimeWindowFields, + ...WebAnalyticsFilterFields, + steps: Schema.Array(FunnelStep), + keyBy: FunnelKeyBy, + windowSeconds: PositiveInt, +}) + +const ProductEventsFunnelBreakdownInputSchema = Schema.Struct({ + ...ProductEventsFunnelInputSchema.fields, + breakdownBy: FunnelBreakdownBy, + limit: Schema.optional(PositiveInt), +}) + +const ProductEventNamesInputSchema = Schema.Struct({ + ...TimeWindowFields, + ...WebAnalyticsFilterFields, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventsFunnelInput = (typeof ProductEventsFunnelInputSchema)["Encoded"] +export type GetProductEventsFunnelBreakdownInput = (typeof ProductEventsFunnelBreakdownInputSchema)["Encoded"] +export type GetProductEventNamesInput = (typeof ProductEventNamesInputSchema)["Encoded"] + +/** One funnel step's result, in step order. */ +export interface FunnelStepCount { + /** 1-based. */ + step: number + count: number +} + +export interface FunnelBreakdownRow { + group: string + step: number + count: number +} + +export interface ProductEventName { + eventName: string + /** `navigation` for page views, `custom` for `track()` calls, `screen` for mobile screens. */ + kind: string + count: number + sessions: number + persons: number +} + +export function getProductEventsFunnel({ data }: { data: GetProductEventsFunnelInput }) { + return getProductEventsFunnelEffect({ data }) +} + +const getProductEventsFunnelEffect = Effect.fn("QueryEngine.getProductEventsFunnel")(function* ({ + data, +}: { + data: GetProductEventsFunnelInput +}) { + const input = yield* decodeInput(ProductEventsFunnelInputSchema, data, "getProductEventsFunnel") + + const result = yield* runWarehouseQuery("productEventsFunnel", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsFunnel({ + payload: new ProductEventsFunnelRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + +export function getProductEventsFunnelBreakdown({ data }: { data: GetProductEventsFunnelBreakdownInput }) { + return getProductEventsFunnelBreakdownEffect({ data }) +} + +const getProductEventsFunnelBreakdownEffect = Effect.fn("QueryEngine.getProductEventsFunnelBreakdown")( + function* ({ data }: { data: GetProductEventsFunnelBreakdownInput }) { + const input = yield* decodeInput( + ProductEventsFunnelBreakdownInputSchema, + data, + "getProductEventsFunnelBreakdown", + ) + + const result = yield* runWarehouseQuery("productEventsFunnelBreakdown", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsFunnelBreakdown({ + payload: new ProductEventsFunnelBreakdownRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } + }, +) + +export function getProductEventNames({ data }: { data: GetProductEventNamesInput }) { + return getProductEventNamesEffect({ data }) +} + +const getProductEventNamesEffect = Effect.fn("QueryEngine.getProductEventNames")(function* ({ + data, +}: { + data: GetProductEventNamesInput +}) { + const input = yield* decodeInput(ProductEventNamesInputSchema, data, "getProductEventNames") + + const result = yield* runWarehouseQuery("productEventNames", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventNames({ + payload: new ProductEventNamesRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + +// Dashboard funnel widget (route data source `product_events_funnel`). +// +// The widget's stored `display.funnel` definition — steps, key, window — is the +// route's params bag; the dashboard planner adds `startTime`/`endTime`. The rows +// come back as `{ name, value }` so the same funnel chart the group-by breakdown +// feeds draws them unchanged: one bar per step, labelled by the step. + +const ProductEventsFunnelWidgetInputSchema = Schema.Struct({ + ...TimeWindowFields, + steps: Schema.Array(FunnelStep), + keyBy: Schema.optional(FunnelKeyBy), + windowSeconds: Schema.optional(PositiveInt), +}) + +export type GetProductEventsFunnelWidgetInput = (typeof ProductEventsFunnelWidgetInputSchema)["Encoded"] + +/** The default key and window a widget without them runs with — same as the /analytics view. */ +const WIDGET_DEFAULT_KEY_BY = "person" +const WIDGET_DEFAULT_WINDOW_SECONDS = 24 * 3600 + +export function getProductEventsFunnelWidget({ data }: { data: GetProductEventsFunnelWidgetInput }) { + return getProductEventsFunnelWidgetEffect({ data }) +} + +const getProductEventsFunnelWidgetEffect = Effect.fn("QueryEngine.getProductEventsFunnelWidget")(function* ({ + data, +}: { + data: GetProductEventsFunnelWidgetInput +}) { + const input = yield* decodeInput( + ProductEventsFunnelWidgetInputSchema, + data, + "getProductEventsFunnelWidget", + ) + + const result = yield* runWarehouseQuery("productEventsFunnelWidget", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsFunnel({ + payload: new ProductEventsFunnelRequest({ + startTime: input.startTime, + endTime: input.endTime, + steps: input.steps, + keyBy: input.keyBy ?? WIDGET_DEFAULT_KEY_BY, + windowSeconds: input.windowSeconds ?? WIDGET_DEFAULT_WINDOW_SECONDS, + }), + }) + }), + ) + + const countByStep = new Map(result.data.map((row) => [row.step, row.count])) + return { + data: input.steps.map((step, index) => ({ + name: funnelStepLabel(step), + value: countByStep.get(index + 1) ?? 0, + })), + } +}) diff --git a/apps/web/src/api/warehouse/web-analytics.ts b/apps/web/src/api/warehouse/web-analytics.ts index 3f66d9604..48c5bfd71 100644 --- a/apps/web/src/api/warehouse/web-analytics.ts +++ b/apps/web/src/api/warehouse/web-analytics.ts @@ -14,7 +14,7 @@ import { import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "@/api/warehouse/effect-utils" -const WebAnalyticsFilterFields = { +export const WebAnalyticsFilterFields = { host: Schema.optional(Schema.String), pagePath: Schema.optional(Schema.String), referrerHost: Schema.optional(Schema.String), @@ -29,7 +29,7 @@ const WebAnalyticsFilterFields = { visitorType: Schema.optional(Schema.Literals(["new", "returning"])), } as const -const TimeWindowFields = { +export const TimeWindowFields = { startTime: WarehouseDateTimeString, endTime: WarehouseDateTimeString, } as const diff --git a/apps/web/src/components/ai-elements/renderers/tool-renderer.tsx b/apps/web/src/components/ai-elements/renderers/tool-renderer.tsx index 7bf44a2bd..14f91e63c 100644 --- a/apps/web/src/components/ai-elements/renderers/tool-renderer.tsx +++ b/apps/web/src/components/ai-elements/renderers/tool-renderer.tsx @@ -493,6 +493,79 @@ export function ToolRenderer({ data: output }: { data: StructuredToolOutput }) { /> ) + + case "query_funnel": { + const first = output.data.steps[0]?.count ?? 0 + const last = output.data.steps[output.data.steps.length - 1]?.count ?? 0 + return ( + + + [ + String(step.step), + step.label, + String(step.count), + `${(step.ofFirst * 100).toFixed(1)}%`, + step.ofPrevious === null ? "—" : `${(step.ofPrevious * 100).toFixed(1)}%`, + step.step === 1 ? "—" : `-${step.dropOff}`, + ]), + title: `Funnel · ${output.data.steps.length} steps · within ${output.data.windowSeconds}s`, + }} + /> + {output.data.breakdown ? ( + `Step ${step.step}`), + "Conv.", + ], + rows: output.data.breakdown.groups.map((group) => [ + group.group === "" ? "(none)" : group.group, + ...group.counts.map((count) => String(count)), + group.conversion === null + ? "—" + : `${(group.conversion * 100).toFixed(1)}%`, + ]), + title: `By ${output.data.breakdown.by}`, + }} + /> + ) : null} + + ) + } + + case "list_product_events": + return ( + [ + event.eventName, + event.kind, + String(event.count), + String(event.sessions), + String(event.persons), + ]), + title: `Product events · ${output.data.events.length}`, + }} + /> + ) } return ( diff --git a/apps/web/src/components/dashboard-builder/config/settings-fields.tsx b/apps/web/src/components/dashboard-builder/config/settings-fields.tsx index 975fc0e43..d7bbff390 100644 --- a/apps/web/src/components/dashboard-builder/config/settings-fields.tsx +++ b/apps/web/src/components/dashboard-builder/config/settings-fields.tsx @@ -16,9 +16,16 @@ import { PANEL_TYPES, fromPanelType, toPanelType } from "@/lib/query-builder/pan import { STAT_AGGREGATES, toSeriesFieldOptions, + type FunnelWidgetDraft, type QueryBuilderWidgetState, type StatAggregate, } from "@/lib/query-builder/widget-builder-shared" +import { formatWarehouseDateTime } from "@maple/query-engine" +import { Result } from "@/lib/effect-atom" +import { productEventNamesResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { useRetainedRefreshableResultValue } from "@/hooks/use-retained-refreshable-result-value" +import { FunnelStepBuilder } from "@/components/funnels/funnel-step-builder" +import { FUNNEL_KEY_BY_OPTIONS, FUNNEL_WINDOW_OPTIONS } from "@/components/funnels/definition" // The settings rail's vocabulary. // @@ -679,11 +686,112 @@ function WidgetTimeRange() { ) } +/** + * The funnel widget's product-event definition: steps, what to count, and the + * conversion window. Leaving the steps empty keeps the widget on its query set + * (a group-by breakdown drawn as a funnel); adding one switches it to the funnel + * endpoint and the query builder on the left stops being what it fetches. + */ +function FunnelSteps() { + const { state, set } = useSettings() + const { + state: { resolvedTimeRange }, + } = useDashboardTimeRange() + const funnel = state.funnel + const usesSteps = funnel.steps.length > 0 + + // Suggestions over the dashboard's window; a fresh org's builder shows none + // and the inputs stay free-text. + const eventNamesResult = useRetainedRefreshableResultValue( + productEventNamesResultAtom({ + data: { + startTime: + resolvedTimeRange?.startTime ?? formatWarehouseDateTime(Date.now() - 7 * 24 * 3_600_000), + endTime: resolvedTimeRange?.endTime ?? formatWarehouseDateTime(Date.now()), + limit: 200, + }, + }), + ) + const eventNames = Result.builder(eventNamesResult) + .onSuccess((rows) => + rows.data + .filter((row) => row.kind !== "navigation") + .map((row) => ({ name: row.eventName, count: row.count })), + ) + .orElse(() => []) + + const update = (patch: Partial) => set({ funnel: { ...funnel, ...patch } }) + + return ( + <> + +

+ {usesSteps + ? "Counting product events per step. The query set on the left is not used." + : "Leave empty to draw the query's group-by rows as a funnel, or add product-event steps."} +

+ update({ steps: [...steps] })} + eventNames={eventNames} + compact + /> + + {usesSteps ? ( + <> + + update({ keyBy })} + options={FUNNEL_KEY_BY_OPTIONS.map((option) => ({ + value: option.value, + label: option.label, + }))} + /> + + + + + + ) : null} + + ) +} + /** * The rail's field vocabulary. A panel type's `ConfigPanel` composes these; none * of them takes the widget state as a prop. */ export const WidgetSettings = { + FunnelSteps, Divider, Name, Description, diff --git a/apps/web/src/components/dashboard-builder/data-source-registry.ts b/apps/web/src/components/dashboard-builder/data-source-registry.ts index a17b66e41..3c35a4c79 100644 --- a/apps/web/src/components/dashboard-builder/data-source-registry.ts +++ b/apps/web/src/components/dashboard-builder/data-source-registry.ts @@ -23,6 +23,7 @@ import { getQueryBuilderTimeseries } from "@/api/warehouse/query-builder-timeser import { getQueryBuilderBreakdown } from "@/api/warehouse/query-builder-breakdown" import { getQueryBuilderList } from "@/api/warehouse/query-builder-list" import { getRawSqlChart } from "@/api/warehouse/raw-sql-chart" +import { getProductEventsFunnelWidget } from "@/api/warehouse/product-events" /** * Error channel shared by every warehouse server function. They fail with the @@ -66,6 +67,7 @@ export const serverFunctionMap: Record = { custom_query_builder_breakdown: getQueryBuilderBreakdown, custom_query_builder_list: getQueryBuilderList, raw_sql_chart: getRawSqlChart, + product_events_funnel: getProductEventsFunnelWidget, markdown_static: markdownStaticServerFn, } satisfies Record diff --git a/apps/web/src/components/dashboard-builder/types.ts b/apps/web/src/components/dashboard-builder/types.ts index 20f8f3ed8..65ed78521 100644 --- a/apps/web/src/components/dashboard-builder/types.ts +++ b/apps/web/src/components/dashboard-builder/types.ts @@ -60,6 +60,7 @@ export type DataSourceEndpoint = | "custom_query_builder_breakdown" | "custom_query_builder_list" | "raw_sql_chart" + | "product_events_funnel" | "markdown_static" // A straight alias of the schema type, as of v3. diff --git a/apps/web/src/components/dashboard-builder/widgets/types/breakdown.tsx b/apps/web/src/components/dashboard-builder/widgets/types/breakdown.tsx index 98c8cd314..de40eced9 100644 --- a/apps/web/src/components/dashboard-builder/widgets/types/breakdown.tsx +++ b/apps/web/src/components/dashboard-builder/widgets/types/breakdown.tsx @@ -1,5 +1,5 @@ import { WIDGET_TYPES } from "@maple/domain/http" -import { makeQueryDataSource } from "@maple/widgets/dashboard" +import { makeProductEventsFunnelDataSource, makeQueryDataSource } from "@maple/widgets/dashboard" import { ArrowTrendDownIcon, @@ -31,9 +31,15 @@ import { BREAKDOWN_TAIL_LIMIT } from "@maple/query-engine/query-builder" import type { BuildDataSourceContext } from "@/lib/query-builder/widget-builder-shared" import { hasActiveGroupBy, + hasFunnelSteps, histogramValueColumn, parsePositiveNumber, } from "@/lib/query-builder/widget-builder-shared" +import { + DEFAULT_FUNNEL_KEY_BY, + DEFAULT_FUNNEL_WINDOW_SECONDS, + completedSteps, +} from "@/components/funnels/definition" import type { WidgetDataSource } from "@/components/dashboard-builder/types" import { chartPresetPreview } from "@/components/dashboard-builder/widgets/types/preset-preview" @@ -85,17 +91,78 @@ export const pieWidgetType: WidgetTypeDefinition = { buildDisplay: ({ base }) => base, } +/** + * Two funnels share one visualization. Without product-event steps it is the + * original: a group-by breakdown drawn as descending stages. With them + * (`display.funnel.steps`) it is a conversion funnel over `product_events`, + * fetched through the `product_events_funnel` route instead of the query set — + * same renderer, same `{ name, value }` rows, one bar per step. The definition + * is persisted on the display block (additive: older readers of the document + * see a funnel with extra keys they ignore) and mirrored into the route params + * so the fetch path never has to read the display. + */ export const funnelWidgetType: WidgetTypeDefinition = { meta: WIDGET_TYPES.funnel, // A funnel is a descending series of stages. icon: ArrowTrendDownIcon, Renderer: FunnelWidget, queryEditor: "builder", - ConfigPanel: () => , + ConfigPanel: () => ( + <> + + + + + ), presets: funnelPresets, PresetPreview: chartPresetPreview("query-builder-funnel"), - buildDataSource: breakdownDataSource, - buildDisplay: ({ base }) => base, + + initialState: (widget) => { + const stored = widget.display.funnel + return { + funnel: { + steps: [...(stored?.steps ?? [])], + keyBy: stored?.keyBy ?? DEFAULT_FUNNEL_KEY_BY, + windowSeconds: stored?.windowSeconds ?? DEFAULT_FUNNEL_WINDOW_SECONDS, + ...(stored?.breakdownBy !== undefined ? { breakdownBy: stored.breakdownBy } : undefined), + }, + } + }, + + ownsDataSource: hasFunnelSteps, + + buildDataSource: (ctx) => + hasFunnelSteps(ctx.state) + ? makeProductEventsFunnelDataSource(ctx.state.funnel, ctx.sharedTransform) + : breakdownDataSource(ctx), + + buildDisplay: ({ base, state, widget }) => + extendDisplay(base, { + funnel: hasFunnelSteps(state) + ? { + ...widget.display.funnel, + steps: state.funnel.steps, + keyBy: state.funnel.keyBy, + windowSeconds: state.funnel.windowSeconds, + ...(state.funnel.breakdownBy !== undefined + ? { breakdownBy: state.funnel.breakdownBy } + : undefined), + } + : // Steps removed: drop the definition, keep the rendering flags. + widget.display.funnel && widget.display.funnel.showStepPercent !== undefined + ? { showStepPercent: widget.display.funnel.showStepPercent } + : undefined, + }), + + validate: ({ state }) => { + if (!hasFunnelSteps(state)) return null + const incomplete = state.funnel.steps.findIndex((step) => completedSteps([step]).length === 0) + if (incomplete !== -1) return `Step ${incomplete + 1} needs an event name, page path or session value` + if (state.funnel.steps.some((step, index) => index > 0 && step.kind === "session")) { + return "A session step is only valid as step 1" + } + return null + }, } /** diff --git a/apps/web/src/components/dashboard-builder/widgets/widget-type-registry.ts b/apps/web/src/components/dashboard-builder/widgets/widget-type-registry.ts index 587a10de7..b3b258f42 100644 --- a/apps/web/src/components/dashboard-builder/widgets/widget-type-registry.ts +++ b/apps/web/src/components/dashboard-builder/widgets/widget-type-registry.ts @@ -97,6 +97,13 @@ export interface WidgetTypeDefinition { buildDisplay: (ctx: BuildDisplayContext) => WidgetDisplayConfig /** Returns a message that blocks Apply, or `null`. Shared rules run first. */ validate?: (ctx: ValidateContext) => string | null + /** + * True when, for this state, the type fetches through a source of its own + * rather than the query set — a funnel with product-event steps. The shared + * query validation (at least one query, group-by required, …) is skipped and + * only `validate` runs. + */ + ownsDataSource?: (state: QueryBuilderWidgetState) => boolean } /** diff --git a/apps/web/src/components/funnels/analytics-funnels-view.tsx b/apps/web/src/components/funnels/analytics-funnels-view.tsx new file mode 100644 index 000000000..7beaefc40 --- /dev/null +++ b/apps/web/src/components/funnels/analytics-funnels-view.tsx @@ -0,0 +1,396 @@ +import type { ReactNode } from "react" +import { Result } from "@/lib/effect-atom" + +import { Input } from "@maple/ui/components/ui/input" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" + +import { QueryErrorState } from "@/components/common/query-error-state" +import { ArrowTrendDownIcon, CodeIcon } from "@/components/icons" +import type { AnalyticsFilters } from "@/components/analytics/filters" +import { + productEventNamesResultAtom, + productEventsFunnelBreakdownResultAtom, + productEventsFunnelResultAtom, + webAnalyticsPagesResultAtom, +} from "@/lib/services/atoms/warehouse-query-atoms" +import { useRetainedRefreshableResultValue } from "@/hooks/use-retained-refreshable-result-value" + +import { + FUNNEL_KEY_BY_OPTIONS, + FUNNEL_SESSION_DIMENSIONS, + FUNNEL_SESSION_DIMENSION_LABEL, + FUNNEL_WINDOW_OPTIONS, + completedSteps, + stepLabel, + type FunnelBreakdownBy, + type FunnelDefinition, + type FunnelKeyBy, + type FunnelSessionDimension, +} from "./definition" +import { FunnelStepBuilder } from "./funnel-step-builder" +import { FunnelBreakdownTable, FunnelResults } from "./funnel-results" + +// The Funnels view of /analytics. The definition lives in the URL (the route +// owns it and hands it down with a setter), so a funnel is a shareable link and +// the back button walks through edits. The sidebar filters come along as the +// population filter — the same object the Overview panels are narrowed by. + +const PAGE_SUGGESTION_LIMIT = 100 +const BREAKDOWN_LIMIT = 10 + +const ATTRIBUTE_PREFIX = "attribute:" +const NONE = "__none__" +const ATTRIBUTE = "__attribute__" + +const KEY_BY_NOUN = { + person: "persons", + visitor: "visitors", + user: "users", + session: "sessions", +} satisfies Record + +interface AnalyticsFunnelsViewProps { + startTime: string + endTime: string + filters: AnalyticsFilters + definition: FunnelDefinition + onDefinitionChange: (definition: FunnelDefinition) => void +} + +export function AnalyticsFunnelsView({ + startTime, + endTime, + filters, + definition, + onDefinitionChange, +}: AnalyticsFunnelsViewProps) { + const windowInput = { startTime, endTime, ...filters } + + const eventNamesResult = useRetainedRefreshableResultValue( + productEventNamesResultAtom({ data: { ...windowInput, limit: 200 } }), + ) + const pagesResult = useRetainedRefreshableResultValue( + webAnalyticsPagesResultAtom({ data: { ...windowInput, limit: PAGE_SUGGESTION_LIMIT } }), + ) + + const eventNames = Result.builder(eventNamesResult) + .onSuccess((rows) => rows.data) + .orElse(() => []) + // The picker lists `track()` events; page views are the Page step's business. + const customEvents = eventNames.filter((row) => row.kind !== "navigation") + const eventSuggestions = customEvents.map((row) => ({ name: row.eventName, count: row.count })) + const pageSuggestions = Result.builder(pagesResult) + .onSuccess((rows) => rows.data.map((page) => ({ name: page.pagePath, count: page.pageViews }))) + .orElse(() => []) + + // Only complete steps go to the warehouse — a step still being typed would + // otherwise fire a query per keystroke and 400 on the blank name. + const steps = completedSteps(definition.steps) + const labels = steps.map(stepLabel) + const unitNoun = KEY_BY_NOUN[definition.keyBy] + + const set = (patch: Partial) => onDefinitionChange({ ...definition, ...patch }) + + return ( +
+
+
+
+ Steps + + in order · session step only first · up to 10 + +
+
+ + + + + + + + onDefinitionChange( + breakdownBy === undefined + ? { + steps: definition.steps, + keyBy: definition.keyBy, + windowSeconds: definition.windowSeconds, + } + : { ...definition, breakdownBy }, + ) + } + /> +
+
+
+ set({ steps: next })} + eventNames={eventSuggestions} + pagePaths={pageSuggestions} + /> +
+
+ + {Result.isSuccess(eventNamesResult) && customEvents.length === 0 ? ( + + ) : null} + + {steps.length === 0 ? ( + + + + + + Add a step to see conversion + + Pick an event, a page, or — for step 1 — how the session was acquired. Each + further step is counted only for {unitNoun} who did the previous one first, inside + the window. + + + + ) : ( + + )} +
+ ) +} + +function FunnelQueries({ + input, + labels, + unitNoun, + breakdownBy, +}: { + input: Parameters[0]["data"] + labels: ReadonlyArray + unitNoun: string + breakdownBy: FunnelBreakdownBy | undefined +}) { + const funnelResult = useRetainedRefreshableResultValue(productEventsFunnelResultAtom({ data: input })) + return ( + <> + {Result.builder(funnelResult) + .onInitial(() => ( + <> + + + + )) + .onError((error) => ) + .onSuccess((rows, result) => ( + + )) + .render()} + {breakdownBy !== undefined ? ( + + ) : null} + + ) +} + +function FunnelBreakdownQuery({ + input, + labels, + breakdownBy, +}: { + input: Parameters[0]["data"] + labels: ReadonlyArray + breakdownBy: FunnelBreakdownBy +}) { + const breakdownResult = useRetainedRefreshableResultValue( + productEventsFunnelBreakdownResultAtom({ data: { ...input, breakdownBy, limit: BREAKDOWN_LIMIT } }), + ) + return Result.builder(breakdownResult) + .onInitial(() => ) + .onError((error) => ) + .onSuccess((rows, result) => ( + + )) + .render() +} + +function LabelledSelect({ label, children }: { label: string; children: ReactNode }) { + return ( +
+ {label} + {children} +
+ ) +} + +/** + * Breakdown: none, one of the session dimensions, or `attribute:` typed + * by hand — the attribute keys on `track()` events are the customer's own + * vocabulary and there is no cheap way to list them. + */ +function BreakdownPicker({ + value, + onChange, +}: { + value: FunnelBreakdownBy | undefined + onChange: (value: FunnelBreakdownBy | undefined) => void +}) { + const isAttribute = value !== undefined && value.startsWith(ATTRIBUTE_PREFIX) + const selected = value === undefined ? NONE : isAttribute ? ATTRIBUTE : value + const attributeKey = isAttribute ? value.slice(ATTRIBUTE_PREFIX.length) : "" + const items = { + [NONE]: "None", + ...FUNNEL_SESSION_DIMENSION_LABEL, + [ATTRIBUTE]: "Attribute…", + } + return ( + + + {isAttribute ? ( + onChange(`${ATTRIBUTE_PREFIX}${event.target.value}`)} + placeholder="attribute key, e.g. plan" + aria-label="Breakdown attribute key" + className="w-40 font-mono text-xs" + /> + ) : null} + + ) +} + +/** + * Shown when nothing but page views has arrived: the funnel still works over + * pages, but the reason to open this tab is `track()`, so say how to start. + */ +function NoCustomEventsCallout() { + return ( +
+ + + +
+

No custom events in this window

+

+ Page steps work from the page views you already send. To measure signups, checkouts and + the steps in between, call{" "} + + maple.track("signup_completed", {'{ plan: "pro" }'}) + {" "} + from the browser SDK, or{" "} + + MapleEvents.track() + {" "} + server-side — each name shows up here as a step. +

+
+
+ ) +} diff --git a/apps/web/src/components/funnels/conversion.test.ts b/apps/web/src/components/funnels/conversion.test.ts new file mode 100644 index 000000000..34c94ed4e --- /dev/null +++ b/apps/web/src/components/funnels/conversion.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest" +import { funnelStepStats, groupBreakdownRows, overallConversion } from "./conversion" + +describe("funnelStepStats", () => { + it("derives share-of-first, step conversion and drop-off per step", () => { + const stats = funnelStepStats( + ["Landed", "Signed up", "Paid"], + [ + { step: 1, count: 200 }, + { step: 2, count: 50 }, + { step: 3, count: 10 }, + ], + ) + expect(stats.map((s) => s.count)).toEqual([200, 50, 10]) + expect(stats.map((s) => s.ofFirst)).toEqual([1, 0.25, 0.05]) + expect(stats.map((s) => s.ofPrevious)).toEqual([null, 0.25, 0.2]) + expect(stats.map((s) => s.dropOff)).toEqual([0, 150, 40]) + expect(stats.map((s) => s.dropOffRate)).toEqual([null, 0.75, 0.8]) + expect(overallConversion(stats)).toBe(0.05) + }) + + it("matches rows by step number and treats a missing step as zero", () => { + const stats = funnelStepStats(["a", "b", "c"], [{ step: 1, count: 4 }]) + expect(stats.map((s) => s.count)).toEqual([4, 0, 0]) + // Step 3's previous step counted nobody, so its conversion is unknown, not 0%. + expect(stats[2]!.ofPrevious).toBeNull() + expect(stats[2]!.dropOffRate).toBeNull() + expect(stats[1]!.ofPrevious).toBe(0) + }) + + it("reports no overall conversion for an empty first step or a single step", () => { + expect(overallConversion(funnelStepStats(["a", "b"], []))).toBeNull() + expect(overallConversion(funnelStepStats(["a"], [{ step: 1, count: 9 }]))).toBeNull() + }) +}) + +describe("groupBreakdownRows", () => { + it("pivots group/step rows into one row per group, keeping first-seen order", () => { + const grouped = groupBreakdownRows(3, [ + { group: "google.com", step: 1, count: 30 }, + { group: "google.com", step: 2, count: 12 }, + { group: "(direct)", step: 1, count: 20 }, + { group: "google.com", step: 3, count: 4 }, + // Out-of-range steps are ignored rather than thrown. + { group: "(direct)", step: 7, count: 99 }, + ]) + expect(grouped).toEqual([ + { group: "google.com", counts: [30, 12, 4] }, + { group: "(direct)", counts: [20, 0, 0] }, + ]) + }) +}) diff --git a/apps/web/src/components/funnels/conversion.ts b/apps/web/src/components/funnels/conversion.ts new file mode 100644 index 000000000..7acb6afd5 --- /dev/null +++ b/apps/web/src/components/funnels/conversion.ts @@ -0,0 +1,71 @@ +// Pure funnel arithmetic — the numbers every surface prints beside a step. + +export interface FunnelStepStat { + /** 1-based step index. */ + readonly step: number + readonly label: string + readonly count: number + /** Share of step 1, 0–1. Step 1 itself is 1 (or 0 when it counted nobody). */ + readonly ofFirst: number + /** Conversion from the previous step, 0–1; `null` on step 1 or when the previous step counted nobody. */ + readonly ofPrevious: number | null + /** How many the previous step lost to reach this one; 0 on step 1. */ + readonly dropOff: number + /** `dropOff` as a share of the previous step, 0–1; `null` on step 1 or when the previous step counted nobody. */ + readonly dropOffRate: number | null +} + +/** + * Join the query's `{ step, count }` rows to their labels and derive the + * conversion columns. Rows are matched by step number, not position, and a step + * the query did not return counts zero — `windowFunnel` gives one row per level + * so that is defensive rather than expected. + */ +export function funnelStepStats( + labels: ReadonlyArray, + rows: ReadonlyArray<{ readonly step: number; readonly count: number }>, +): ReadonlyArray { + const countByStep = new Map() + for (const row of rows) countByStep.set(row.step, row.count) + const first = countByStep.get(1) ?? 0 + return labels.map((label, index) => { + const step = index + 1 + const count = countByStep.get(step) ?? 0 + const previous = index === 0 ? null : (countByStep.get(step - 1) ?? 0) + const dropOff = previous === null ? 0 : Math.max(0, previous - count) + return { + step, + label, + count, + ofFirst: index === 0 ? (first > 0 ? 1 : 0) : first > 0 ? count / first : 0, + ofPrevious: previous === null || previous <= 0 ? null : count / previous, + dropOff, + dropOffRate: previous === null || previous <= 0 ? null : dropOff / previous, + } + }) +} + +/** Overall conversion: last step over first, 0–1; `null` with fewer than two steps or an empty first step. */ +export function overallConversion(stats: ReadonlyArray): number | null { + if (stats.length < 2) return null + const first = stats[0]!.count + if (first <= 0) return null + return stats[stats.length - 1]!.count / first +} + +/** Group breakdown rows into one `{ group, counts[] }` per group, in the order groups first appear. */ +export function groupBreakdownRows( + stepCount: number, + rows: ReadonlyArray<{ readonly group: string; readonly step: number; readonly count: number }>, +): ReadonlyArray<{ readonly group: string; readonly counts: ReadonlyArray }> { + const byGroup = new Map() + for (const row of rows) { + let counts = byGroup.get(row.group) + if (!counts) { + counts = new Array(stepCount).fill(0) + byGroup.set(row.group, counts) + } + if (row.step >= 1 && row.step <= stepCount) counts[row.step - 1] = row.count + } + return [...byGroup.entries()].map(([group, counts]) => ({ group, counts })) +} diff --git a/apps/web/src/components/funnels/definition.ts b/apps/web/src/components/funnels/definition.ts new file mode 100644 index 000000000..a399d6fb1 --- /dev/null +++ b/apps/web/src/components/funnels/definition.ts @@ -0,0 +1,132 @@ +// The funnel definition vocabulary shared by the /analytics Funnels view, the +// dashboard funnel widget and the step builder they both mount. +// +// A definition is `steps` + `keyBy` + `windowSeconds` (+ optional +// `breakdownBy`), exactly the option bag `productEventsFunnelQuery` takes; the +// wire schemas live in `@maple/query-model` and are reused here unchanged so a +// URL, a widget's stored config and an API request all decode the same shape. + +import { Schema } from "effect" +import { + FUNNEL_MAX_STEPS, + FUNNEL_SESSION_DIMENSION_LABEL, + FunnelBreakdownBy, + FunnelKeyBy, + FunnelSessionDimension, + FunnelStep, + funnelStepLabel, + type FunnelBreakdownBy as FunnelBreakdownByType, + type FunnelKeyBy as FunnelKeyByType, + type FunnelSessionDimension as FunnelSessionDimensionType, + type FunnelStep as FunnelStepType, +} from "@maple/query-model" + +export type { FunnelStepType as FunnelStep, FunnelKeyByType as FunnelKeyBy } +export type { + FunnelBreakdownByType as FunnelBreakdownBy, + FunnelSessionDimensionType as FunnelSessionDimension, +} + +export { FUNNEL_MAX_STEPS } + +export interface FunnelDefinition { + readonly steps: ReadonlyArray + readonly keyBy: FunnelKeyByType + readonly windowSeconds: number + readonly breakdownBy?: FunnelBreakdownByType +} + +export const DEFAULT_FUNNEL_KEY_BY: FunnelKeyByType = "person" +export const DEFAULT_FUNNEL_WINDOW_SECONDS = 24 * 3600 + +export const FUNNEL_WINDOW_OPTIONS: ReadonlyArray<{ readonly value: number; readonly label: string }> = [ + { value: 3600, label: "1 hour" }, + { value: 24 * 3600, label: "24 hours" }, + { value: 7 * 24 * 3600, label: "7 days" }, + { value: 30 * 24 * 3600, label: "30 days" }, +] + +export const FUNNEL_KEY_BY_OPTIONS: ReadonlyArray<{ + readonly value: FunnelKeyByType + readonly label: string + readonly description: string +}> = [ + { + value: "person", + label: "Person", + description: + "User id when known, else the visitor — anonymous and signed-in activity collapse into one.", + }, + { value: "visitor", label: "Visitor", description: "The browser's anonymous visitor id." }, + { value: "user", label: "User", description: "Identified users only." }, + { + value: "session", + label: "Session", + description: "Each session on its own; server events take no part.", + }, +] + +export { FUNNEL_SESSION_DIMENSION_LABEL } + +export const FUNNEL_SESSION_DIMENSIONS: ReadonlyArray = + FunnelSessionDimension.literals + +/** URL search-param fields for the /analytics Funnels view. Spread into `validateSearch`. */ +export const funnelSearchFields = { + view: Schema.optional(Schema.Literals(["overview", "funnels"])), + steps: Schema.optional(Schema.Array(FunnelStep)), + keyBy: Schema.optional(FunnelKeyBy), + window: Schema.optional(Schema.Number), + breakdown: Schema.optional(FunnelBreakdownBy), +} as const + +export type AnalyticsView = "overview" | "funnels" + +/** Read a definition off the route's decoded search object, defaults filled in. */ +export const funnelFromSearch = (search: { + readonly steps?: ReadonlyArray + readonly keyBy?: FunnelKeyByType + readonly window?: number + readonly breakdown?: FunnelBreakdownByType +}): FunnelDefinition => ({ + steps: search.steps ?? [], + keyBy: search.keyBy ?? DEFAULT_FUNNEL_KEY_BY, + windowSeconds: + search.window !== undefined && Number.isFinite(search.window) && search.window > 0 + ? search.window + : DEFAULT_FUNNEL_WINDOW_SECONDS, + ...(search.breakdown !== undefined ? { breakdownBy: search.breakdown } : undefined), +}) + +/** A fresh event step; the builder's default when a step is added. */ +export const emptyEventStep = (): FunnelStepType => ({ kind: "event", eventName: "" }) + +/** Steps with something to match on. A blank event name is a step still being typed. */ +export const completedSteps = (steps: ReadonlyArray): ReadonlyArray => + steps.filter((step) => { + switch (step.kind) { + case "event": + return step.eventName.trim() !== "" + case "page": + return step.pagePath.trim() !== "" + case "session": + return step.value.trim() !== "" + } + }) + +/** Human label for a step — the bar label in the chart and the row label in the table. */ +export const stepLabel = funnelStepLabel + +/** What the breakdown select shows for a `breakdownBy` value. */ +export const breakdownLabel = (breakdownBy: FunnelBreakdownByType): string => + breakdownBy.startsWith("attribute:") + ? `attribute ${breakdownBy.slice("attribute:".length)}` + : FUNNEL_SESSION_DIMENSION_LABEL[breakdownBy as FunnelSessionDimensionType] + +/** Encode a definition into the search-param fields (empty steps drop the keys). */ +export const funnelToSearch = (definition: FunnelDefinition) => ({ + steps: definition.steps.length > 0 ? definition.steps : undefined, + keyBy: definition.keyBy === DEFAULT_FUNNEL_KEY_BY ? undefined : definition.keyBy, + window: definition.windowSeconds === DEFAULT_FUNNEL_WINDOW_SECONDS ? undefined : definition.windowSeconds, + breakdown: definition.breakdownBy, +}) diff --git a/apps/web/src/components/funnels/funnel-results.tsx b/apps/web/src/components/funnels/funnel-results.tsx new file mode 100644 index 000000000..4458acae3 --- /dev/null +++ b/apps/web/src/components/funnels/funnel-results.tsx @@ -0,0 +1,225 @@ +import { Suspense } from "react" + +import { cn } from "@maple/ui/lib/utils" +import { formatNumber, formatPercent } from "@maple/ui/lib/format" +import { QueryBuilderFunnelChart } from "@maple/ui/components/charts/funnel/query-builder-funnel-chart" +import { ChartSkeleton } from "@maple/ui/components/charts/_shared/chart-skeleton" + +import { ColumnHead, DataTable } from "@/components/infra/primitives/data-table" +import { shareBar } from "@/components/infra/primitives/share-bar" +import { funnelStepStats, groupBreakdownRows, overallConversion, type FunnelStepStat } from "./conversion" +import { breakdownLabel, type FunnelBreakdownBy } from "./definition" + +// The results half of a funnel: the same funnel chart the dashboard widget +// draws (reused from the chart registry — it already takes `{ name, value }` +// rows) over a step table with the numbers a chart cannot carry legibly at ten +// steps: count, share of step 1, conversion from the previous step, and the +// drop-off. A breakdown, when asked for, is a small grouped table underneath. + +/** Label line + bar per row in the funnel chart, plus its "+N more" allowance. */ +const CHART_ROW_PX = 28 +const CHART_PAD_PX = 24 + +const fmtPct = (fraction: number | null): string => (fraction === null ? "—" : formatPercent(fraction)) + +interface FunnelResultsProps { + labels: ReadonlyArray + rows: ReadonlyArray<{ readonly step: number; readonly count: number }> + /** What one count means: "persons", "visitors", "users", "sessions". */ + unitNoun: string + waiting?: boolean + className?: string +} + +export function FunnelResults({ labels, rows, unitNoun, waiting = false, className }: FunnelResultsProps) { + const stats = funnelStepStats(labels, rows) + const conversion = overallConversion(stats) + const entered = stats[0]?.count ?? 0 + const completed = stats[stats.length - 1]?.count ?? 0 + const chartRows = stats.map((stat) => ({ name: stat.label, value: stat.count })) + const chartHeight = Math.max(120, stats.length * CHART_ROW_PX + CHART_PAD_PX) + + return ( +
+
+
+
+ + {conversion === null ? "—" : formatPercent(conversion)} + + + {stats.length < 2 ? "conversion needs two steps" : "converted end to end"} + +
+
+ + {formatNumber(entered)} {unitNoun}{" "} + entered + + {stats.length >= 2 ? ( + + {formatNumber(completed)} completed + + ) : null} +
+
+
+ {entered === 0 ? ( +
+ Nobody matched step 1 in this window. +
+ ) : ( + }> + + + )} +
+
+ +
+ +
+
+ ) +} + +function FunnelStepTable({ stats, unitNoun }: { stats: ReadonlyArray; unitNoun: string }) { + const max = stats.reduce((acc, stat) => Math.max(acc, stat.count), 0) + return ( + + + + + + + {stats.map((stat) => ( +
0 ? stat.count / max : 0)} + className="flex items-center gap-4 border-b border-border/40 px-4 py-2 text-[12px] last:border-0" + > + + {stat.step} + + + {stat.label} + + + {formatNumber(stat.count)} + + + {fmtPct(stat.ofFirst)} + + + {fmtPct(stat.ofPrevious)} + + + {stat.step === 1 ? "—" : `−${formatNumber(stat.dropOff)}`} + {stat.dropOffRate !== null ? ( + · {fmtPct(stat.dropOffRate)} + ) : null} + +
+ ))} + + ) +} + +interface FunnelBreakdownTableProps { + labels: ReadonlyArray + breakdownBy: FunnelBreakdownBy + rows: ReadonlyArray<{ readonly group: string; readonly step: number; readonly count: number }> + waiting?: boolean +} + +/** One row per group, one numeric column per step, and the group's end-to-end conversion. */ +export function FunnelBreakdownTable({ + labels, + breakdownBy, + rows, + waiting = false, +}: FunnelBreakdownTableProps) { + const groups = groupBreakdownRows(labels.length, rows) + const maxFirst = groups.reduce((acc, group) => Math.max(acc, group.counts[0] ?? 0), 0) + return ( +
+
+ + By {breakdownLabel(breakdownBy)} + · top {groups.length} by step 1 + + + first non-empty value per person + +
+ + + + {labels.map((_label, index) => ( + + {groups.length === 0 ? ( + No groups — nobody matched step 1 in this window. + ) : ( + groups.map((group) => { + const first = group.counts[0] ?? 0 + const last = group.counts[group.counts.length - 1] ?? 0 + return ( +
0 ? first / maxFirst : 0)} + className="flex items-center gap-4 border-b border-border/40 px-4 py-2 text-[12px] last:border-0" + > + + {group.group === "" ? "(none)" : group.group} + + {group.counts.map((count, index) => ( + 0 && "text-muted-foreground", + index > 0 && index < group.counts.length - 1 && "max-md:hidden", + )} + > + {formatNumber(count)} + + ))} + + {labels.length < 2 || first <= 0 ? "—" : formatPercent(last / first)} + +
+ ) + }) + )} + +
+ ) +} + +const capitalize = (value: string) => (value.length > 0 ? value[0]!.toUpperCase() + value.slice(1) : value) diff --git a/apps/web/src/components/funnels/funnel-step-builder.tsx b/apps/web/src/components/funnels/funnel-step-builder.tsx new file mode 100644 index 000000000..a20a8b37e --- /dev/null +++ b/apps/web/src/components/funnels/funnel-step-builder.tsx @@ -0,0 +1,379 @@ +import { Button } from "@maple/ui/components/ui/button" +import { + Combobox, + ComboboxContent, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, +} from "@maple/ui/components/ui/combobox" +import { Input } from "@maple/ui/components/ui/input" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" +import { cn } from "@maple/ui/lib/utils" +import { formatNumber } from "@maple/ui/lib/format" + +import { ArrowDownIcon, ArrowUpIcon, PlusIcon, XmarkIcon } from "@/components/icons" +import { + FUNNEL_MAX_STEPS, + FUNNEL_SESSION_DIMENSIONS, + FUNNEL_SESSION_DIMENSION_LABEL, + emptyEventStep, + type FunnelSessionDimension, + type FunnelStep, +} from "./definition" + +// The step builder: an ordered list of up to ten steps, each an event name, a +// page path, or — first step only — a session's acquisition dimension. Shared +// by the /analytics Funnels view and the dashboard funnel widget's config rail, +// so it owns no fetching: the event-name and page-path suggestions come in as +// props and every edit goes straight back out through `onChange`. + +export interface FunnelStepSuggestion { + readonly name: string + /** Shown beside the name when known — event count or page views. */ + readonly count?: number +} + +interface FunnelStepBuilderProps { + steps: ReadonlyArray + onChange: (steps: ReadonlyArray) => void + /** `track()` event names for the event-step autocomplete. */ + eventNames?: ReadonlyArray + /** Page paths for the page-step autocomplete. */ + pagePaths?: ReadonlyArray + /** Tighter spacing for a settings rail. */ + compact?: boolean + className?: string +} + +type StepKind = FunnelStep["kind"] + +const KIND_LABEL = { + event: "Event", + page: "Page", + session: "Session", +} satisfies Record + +/** Change a step's kind, carrying nothing over: the value fields do not mean the same thing. */ +function withKind(step: FunnelStep, kind: StepKind): FunnelStep { + if (step.kind === kind) return step + switch (kind) { + case "event": + return { kind: "event", eventName: "" } + case "page": + return { kind: "page", pagePath: "" } + case "session": + return { kind: "session", dimension: "referrerHost", value: "" } + } +} + +const move = (items: ReadonlyArray, from: number, to: number): ReadonlyArray => { + if (to < 0 || to >= items.length) return items + const next = [...items] + const [item] = next.splice(from, 1) + next.splice(to, 0, item!) + return next +} + +export function FunnelStepBuilder({ + steps, + onChange, + eventNames = [], + pagePaths = [], + compact = false, + className, +}: FunnelStepBuilderProps) { + const update = (index: number, step: FunnelStep) => + onChange(steps.map((current, i) => (i === index ? step : current))) + const remove = (index: number) => onChange(steps.filter((_, i) => i !== index)) + const add = () => onChange([...steps, emptyEventStep()]) + + // A session step is only valid first. Rather than let the query 400, the + // kind is unavailable past step 1 and moving a session step down demotes it + // to an event step at the same position. + const reorder = (from: number, to: number) => { + const next = move(steps, from, to) + onChange(next.map((step, i) => (i > 0 && step.kind === "session" ? withKind(step, "event") : step))) + } + + return ( +
+ {steps.map((step, index) => ( + update(index, next)} + onRemove={() => remove(index)} + onMoveUp={() => reorder(index, index - 1)} + onMoveDown={() => reorder(index, index + 1)} + /> + ))} + +
+ ) +} + +function StepRow({ + index, + step, + total, + compact, + eventNames, + pagePaths, + onChange, + onRemove, + onMoveUp, + onMoveDown, +}: { + index: number + step: FunnelStep + total: number + compact: boolean + eventNames: ReadonlyArray + pagePaths: ReadonlyArray + onChange: (step: FunnelStep) => void + onRemove: () => void + onMoveUp: () => void + onMoveDown: () => void +}) { + const kinds: ReadonlyArray = index === 0 ? ["event", "page", "session"] : ["event", "page"] + const kindItems = Object.fromEntries(kinds.map((kind) => [kind, KIND_LABEL[kind]])) + + return ( +
+ + {index + 1} + + + + +
+ {step.kind === "event" ? ( + onChange({ kind: "event", eventName })} + suggestions={eventNames} + placeholder="Event name, e.g. signup_completed" + ariaLabel={`Step ${index + 1} event name`} + empty="No events recorded in this window — type a name." + /> + ) : step.kind === "page" ? ( + + onChange( + step.host + ? { kind: "page", pagePath, host: step.host } + : { kind: "page", pagePath }, + ) + } + suggestions={pagePaths} + placeholder="Page path, e.g. /pricing" + ariaLabel={`Step ${index + 1} page path`} + empty="No page paths in this window — type a path." + mono + /> + ) : ( + <> + + onChange({ ...step, value: event.target.value })} + placeholder={sessionValuePlaceholder(step.dimension)} + aria-label={`Step ${index + 1} session value`} + className="min-w-0 flex-1 font-mono text-xs" + /> + + )} +
+ +
+ + + +
+
+ ) +} + +function sessionValuePlaceholder(dimension: FunnelSessionDimension): string { + switch (dimension) { + case "referrerHost": + return "e.g. news.ycombinator.com" + case "utmSource": + return "e.g. twitter" + case "utmMedium": + return "e.g. cpc" + case "utmCampaign": + return "e.g. launch-week" + case "country": + return "e.g. DE" + case "host": + return "e.g. app.example.com" + } +} + +/** + * A free-text input with suggestions: the value is whatever was typed, and + * picking a suggestion just types it. Freeform matters — a funnel is often + * built for an event that has not fired yet in this window (or ever, on a + * fresh org), and a strict picker would make that impossible to express. + */ +function SuggestingInput({ + value, + onChange, + suggestions, + placeholder, + ariaLabel, + empty, + mono = false, +}: { + value: string + onChange: (value: string) => void + suggestions: ReadonlyArray + placeholder: string + ariaLabel: string + empty: string + mono?: boolean +}) { + const names = suggestions.map((suggestion) => suggestion.name) + const countOf = new Map(suggestions.map((suggestion) => [suggestion.name, suggestion.count])) + return ( + onChange(next)} + value={names.includes(value) ? value : null} + onValueChange={(next) => { + if (typeof next === "string") onChange(next) + }} + > + 0} + /> + + {empty} + + {(name: string) => ( + + + + {name} + + {countOf.get(name) !== undefined ? ( + + {formatNumber(countOf.get(name)!)} + + ) : null} + + + )} + + + + ) +} diff --git a/apps/web/src/lib/query-builder/widget-builder-shared.ts b/apps/web/src/lib/query-builder/widget-builder-shared.ts index 06c2cc46f..8dfa2cb4d 100644 --- a/apps/web/src/lib/query-builder/widget-builder-shared.ts +++ b/apps/web/src/lib/query-builder/widget-builder-shared.ts @@ -17,7 +17,7 @@ import type { } from "@/components/dashboard-builder/types" import type { LegendPosition } from "@/components/dashboard-builder/config/settings-fields" import { STAT_AGGREGATES, type StatAggregate } from "@maple/domain/http" -import type { QueryComparisonMode } from "@maple/query-model" +import type { FunnelBreakdownBy, FunnelKeyBy, FunnelStep, QueryComparisonMode } from "@maple/query-model" import type { HeatmapColorScale, HeatmapScaleType } from "@maple/domain/http" import { normalizeKey, parseBoolean, parseWhereClause as parseWhereClauses } from "@maple/domain/where-clause" @@ -83,8 +83,27 @@ export interface QueryBuilderWidgetState { heatmapScaleType: HeatmapScaleType // Markdown-specific: the note body. Static — never hits the warehouse. markdownContent: string + /** + * Funnel-specific: the product-event funnel definition. With one or more + * steps the widget is fetched through the funnel endpoint and the query + * builder is bypassed; with none it stays a group-by breakdown drawn as a + * funnel, exactly as before. + */ + funnel: FunnelWidgetDraft } +/** The funnel widget's editor state for its `display.funnel` definition block. */ +export interface FunnelWidgetDraft { + steps: FunnelStep[] + keyBy: FunnelKeyBy + windowSeconds: number + breakdownBy?: FunnelBreakdownBy +} + +/** Whether the funnel state carries a definition the funnel endpoint can run. */ +export const hasFunnelSteps = (state: Pick): boolean => + state.visualization === "funnel" && state.funnel.steps.length > 0 + /** * What a panel type's `buildDataSource` is handed. `base` is the timeseries * data source every query-driven type starts from, already carrying the shared diff --git a/apps/web/src/lib/query-builder/widget-builder-utils.test.ts b/apps/web/src/lib/query-builder/widget-builder-utils.test.ts index eb4522488..2679d0f07 100644 --- a/apps/web/src/lib/query-builder/widget-builder-utils.test.ts +++ b/apps/web/src/lib/query-builder/widget-builder-utils.test.ts @@ -76,6 +76,7 @@ function makeState(): QueryBuilderWidgetState { gaugeMax: "", sparklineEnabled: false, markdownContent: "", + funnel: { steps: [], keyBy: "person", windowSeconds: 86400 }, } } @@ -274,6 +275,108 @@ describe("funnel/heatmap endpoint routing (MAP-49)", () => { }) }) +describe("product-event funnel widget", () => { + const funnelState = (): QueryBuilderWidgetState => ({ + ...makeState(), + visualization: "funnel", + chartId: "query-builder-funnel", + // A placeholder draft with no group-by — what the shared validation would + // reject if it ran; the funnel definition owns the source instead. + queries: [ + { + ...createQueryDraft(0), + groupBy: [], + addOns: { ...createQueryDraft(0).addOns, groupBy: false }, + }, + ], + funnel: { + steps: [ + { kind: "page", pagePath: "/pricing" }, + { kind: "event", eventName: "signup_completed" }, + ], + keyBy: "visitor", + windowSeconds: 3600, + }, + }) + + it("routes to the product_events_funnel route with the definition as params", () => { + const dataSource = buildWidgetDataSource(makeWidget(), funnelState(), ["A"]) + expect(dataSource.kind).toBe("route") + if (dataSource.kind !== "route") throw new Error("expected a route") + expect(dataSource.endpoint).toBe("product_events_funnel") + expect(dataSource.params).toEqual({ + steps: [ + { kind: "page", pagePath: "/pricing" }, + { kind: "event", eventName: "signup_completed" }, + ], + keyBy: "visitor", + windowSeconds: 3600, + }) + }) + + it("stays a group-by breakdown without steps", () => { + const state = { + ...funnelState(), + funnel: { steps: [], keyBy: "person" as const, windowSeconds: 86400 }, + } + expect(routedTo(buildWidgetDataSource(makeWidget(), state, ["A"]))).toBe("breakdown") + }) + + it("persists the definition on display.funnel and reads it back", () => { + const state = funnelState() + const widget = { + ...makeWidget(), + visualization: "funnel" as const, + display: { funnel: { showStepPercent: false } }, + } + const display = buildWidgetDisplay(widget, state) + expect(display.funnel).toEqual({ + showStepPercent: false, + steps: state.funnel.steps, + keyBy: "visitor", + windowSeconds: 3600, + }) + + const reopened = toInitialState({ + ...widget, + display, + dataSource: buildWidgetDataSource(widget, state, ["A"]), + }) + expect(reopened.funnel).toEqual({ steps: state.funnel.steps, keyBy: "visitor", windowSeconds: 3600 }) + }) + + it("drops the definition but keeps the rendering flag when the steps are removed", () => { + const state = { + ...funnelState(), + funnel: { steps: [], keyBy: "person" as const, windowSeconds: 86400 }, + } + const widget = { + ...makeWidget(), + visualization: "funnel" as const, + display: { + funnel: { showStepPercent: true, steps: [{ kind: "event" as const, eventName: "x" }] }, + }, + } + expect(buildWidgetDisplay(widget, state).funnel).toEqual({ showStepPercent: true }) + }) + + it("skips the group-by requirement and validates the steps instead", () => { + expect(validateQueries(funnelState())).toBeNull() + const blank = { + ...funnelState(), + funnel: { + steps: [{ kind: "event" as const, eventName: "" }], + keyBy: "person" as const, + windowSeconds: 1, + }, + } + expect(validateQueries(blank)).toContain("Step 1 needs") + // Without steps the ordinary rule is back: a funnel needs a group-by. + const plain = { ...funnelState(), funnel: { steps: [], keyBy: "person" as const, windowSeconds: 1 } } + expect(validateQueries(plain)).toContain("group-by") + }) +}) + describe("histogram data shape routing", () => { function ungroupedTraceState() { const base = createQueryDraft(0) diff --git a/apps/web/src/lib/query-builder/widget-builder-utils.ts b/apps/web/src/lib/query-builder/widget-builder-utils.ts index d3d0c0859..37aa313f6 100644 --- a/apps/web/src/lib/query-builder/widget-builder-utils.ts +++ b/apps/web/src/lib/query-builder/widget-builder-utils.ts @@ -22,6 +22,7 @@ import { type QueryBuilderWidgetState, } from "@/lib/query-builder/widget-builder-shared" import { WIDGET_TYPES } from "@maple/domain/http" +import { DEFAULT_FUNNEL_KEY_BY, DEFAULT_FUNNEL_WINDOW_SECONDS } from "@/components/funnels/definition" import { dataSourceQuerySet, dataSourceRouteParams, makeQueryDataSource } from "@maple/widgets/dashboard" // Lowering the widget editor's state to a persisted widget, and back. @@ -124,6 +125,7 @@ export function toInitialState(widget: DashboardWidget): QueryBuilderWidgetState heatmapColorScale: "blues", heatmapScaleType: "linear", markdownContent: "", + funnel: { steps: [], keyBy: DEFAULT_FUNNEL_KEY_BY, windowSeconds: DEFAULT_FUNNEL_WINDOW_SECONDS }, ...definition.initialState?.(widget), } @@ -260,6 +262,13 @@ export function validateQueries(state: QueryBuilderWidgetState): string | null { // block Apply on an error the user has no panel to fix. if (definition.queryEditor !== "builder") return null + // A type that has swapped its query set for a source of its own (a funnel + // with product-event steps) validates that source instead: its query drafts + // are the placeholder the state shape requires, not what reaches the warehouse. + if (definition.ownsDataSource?.(state)) { + return definition.validate?.({ state, activeQueries: [], visibleQueries: [] }) ?? null + } + const activeQueries = state.queries.filter((query) => query.enabled !== false) if (activeQueries.length === 0) return "Add at least one query" for (const query of activeQueries) { diff --git a/apps/web/src/lib/query-builder/widget-type-cycle.test.ts b/apps/web/src/lib/query-builder/widget-type-cycle.test.ts index ed80eb5d3..362e0db10 100644 --- a/apps/web/src/lib/query-builder/widget-type-cycle.test.ts +++ b/apps/web/src/lib/query-builder/widget-type-cycle.test.ts @@ -88,6 +88,7 @@ function makeState(overrides: Partial = {}): QueryBuild gaugeMax: "", sparklineEnabled: false, markdownContent: "", + funnel: { steps: [], keyBy: "person", windowSeconds: 86400 }, ...overrides, } } diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index a811130f4..3847ba450 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -117,6 +117,11 @@ import { getWebAnalyticsSummary, getWebAnalyticsTimeseries, } from "@/api/warehouse/web-analytics" +import { + getProductEventNames, + getProductEventsFunnel, + getProductEventsFunnelBreakdown, +} from "@/api/warehouse/product-events" /** * The error union every warehouse server function fails with: the structured @@ -273,6 +278,21 @@ export const webAnalyticsBreakdownsResultAtom = makeQueryAtomFamily(getWebAnalyt staleTime: 30_000, }) +// Product-event funnels share the analytics page and its 30s. The event-name +// list backs the step builder's autocomplete and changes only when someone +// ships a new `track()` call, so it can sit for a minute. +export const productEventsFunnelResultAtom = makeQueryAtomFamily(getProductEventsFunnel, { + staleTime: 30_000, +}) + +export const productEventsFunnelBreakdownResultAtom = makeQueryAtomFamily(getProductEventsFunnelBreakdown, { + staleTime: 30_000, +}) + +export const productEventNamesResultAtom = makeQueryAtomFamily(getProductEventNames, { + staleTime: 60_000, +}) + export const getReplayResultAtom = makeQueryAtomFamily(getReplay, { staleTime: 60_000, }) diff --git a/apps/web/src/routes/analytics/index.tsx b/apps/web/src/routes/analytics/index.tsx index 504aa6b80..64c517852 100644 --- a/apps/web/src/routes/analytics/index.tsx +++ b/apps/web/src/routes/analytics/index.tsx @@ -6,6 +6,7 @@ import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-en import { Button } from "@maple/ui/components/ui/button" import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { ToggleGroup, ToggleGroupItem } from "@maple/ui/components/ui/toggle-group" import { DashboardLayout } from "@/components/layout/dashboard-layout" import { QueryErrorState } from "@/components/common/query-error-state" @@ -49,6 +50,14 @@ import { webAnalyticsSummaryResultAtom, webAnalyticsTimeseriesResultAtom, } from "@/lib/services/atoms/warehouse-query-atoms" +import { AnalyticsFunnelsView } from "@/components/funnels/analytics-funnels-view" +import { + funnelFromSearch, + funnelSearchFields, + funnelToSearch, + type AnalyticsView, + type FunnelDefinition, +} from "@/components/funnels/definition" import { useEffectiveTimeRange } from "@/hooks/use-effective-time-range" import { useRetainedRefreshableResultValue } from "@/hooks/use-retained-refreshable-result-value" import { TimeRangeSearchFields, applyTimeRangeSearch } from "@/components/time-range-picker/search" @@ -58,6 +67,9 @@ import { TimeRangeHeaderControls } from "@/components/time-range-picker/time-ran const analyticsSearchSchema = Schema.Struct({ ...analyticsFilterSearchFields, ...TimeRangeSearchFields, + // `view` picks Overview or Funnels; the funnel definition rides in the URL + // too so a funnel is a shareable link. + ...funnelSearchFields, }) const DEFAULT_PRESET = "7d" @@ -99,16 +111,35 @@ function WebAnalyticsPage() { onFilterChange(key, toggleFilterValue(filters[key], value)) } + // Clearing filters keeps the view and the funnel: those are what you are + // looking at, the filters are how narrowly. const onClearFilters = () => { navigate({ search: { startTime: search.startTime, endTime: search.endTime, timePreset: search.timePreset, + view: search.view, + steps: search.steps, + keyBy: search.keyBy, + window: search.window, + breakdown: search.breakdown, }, }) } + const view: AnalyticsView = search.view ?? "overview" + const onViewChange = (next: AnalyticsView) => { + navigate({ search: (prev) => ({ ...prev, view: next === "overview" ? undefined : next }) }) + } + + const funnel = funnelFromSearch(search) + // An edit per history entry would make Back useless while + // typing an event name, so definition edits replace the current entry. + const onFunnelChange = (definition: FunnelDefinition) => { + navigate({ replace: true, search: (prev) => ({ ...prev, ...funnelToSearch(definition) }) }) + } + // Retained, not bare: the filters are part of every atom key, so each row click // instantiates a fresh atom whose first emission is `Initial`. Reading that // directly would replace the sidebar with a skeleton on every click and reset @@ -136,7 +167,23 @@ function WebAnalyticsPage() { - + { + const next = values[0] + if (next === "overview" || next === "funnels") onViewChange(next) + }} + variant="outline" + size="sm" + aria-label="Analytics view" + > + Overview + Funnels + + } + >
{/* The reciprocal of the Analytics button on Session Replays: this page aggregates the sessions that page plays back one at a time, @@ -175,8 +222,12 @@ function WebAnalyticsPage() {
0 ? (
@@ -201,13 +252,23 @@ function WebAnalyticsPage() { ) : undefined } /> - + {view === "funnels" ? ( + + ) : ( + + )}
diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index e1312e0ae..8e24fd6a7 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -22,6 +22,7 @@ import { import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { warehouseHttpErrors } from "./warehouse" +import { FunnelBreakdownBy, FunnelKeyBy, FunnelStep } from "@maple/query-model" // Dedicated endpoint schemas @@ -1272,6 +1273,89 @@ export class WebAnalyticsBreakdownsResponse extends Schema.Class( + "ProductEventsFunnelRequest", +)(ProductEventsFunnelFields) {} + +export class ProductEventsFunnelResponse extends Schema.Class( + "ProductEventsFunnelResponse", +)({ + /** Exactly one row per step, in step order (1-based `step`). */ + data: Schema.Array(Schema.Struct({ step: Schema.Number, count: Schema.Number })), +}) {} + +export class ProductEventsFunnelBreakdownRequest extends Schema.Class( + "ProductEventsFunnelBreakdownRequest", +)({ + ...ProductEventsFunnelFields, + breakdownBy: FunnelBreakdownBy, + /** Groups to keep, ranked by step-1 count. Default 10, max 20. */ + limit: Schema.optional(Schema.Number), +}) {} + +export class ProductEventsFunnelBreakdownResponse extends Schema.Class( + "ProductEventsFunnelBreakdownResponse", +)({ + data: Schema.Array(Schema.Struct({ group: Schema.String, step: Schema.Number, count: Schema.Number })), +}) {} + +export class ProductEventNamesRequest extends Schema.Class( + "ProductEventNamesRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + /** Default 100. */ + limit: Schema.optional(Schema.Number), + ...WebAnalyticsFilterFields, +}) {} + +export class ProductEventNamesResponse extends Schema.Class( + "ProductEventNamesResponse", +)({ + data: Schema.Array( + Schema.Struct({ + eventName: Schema.String, + /** `navigation` for page views, `custom` for `track()` calls, `screen` for mobile screens. */ + kind: Schema.String, + count: Schema.Number, + sessions: Schema.Number, + persons: Schema.Number, + }), + ), +}) {} + export class PodFacetsRequest extends Schema.Class("PodFacetsRequest")({ startTime: TinybirdDateTime, endTime: TinybirdDateTime, @@ -2121,6 +2205,29 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("productEventsFunnel", "/product-events-funnel", { + payload: ProductEventsFunnelRequest, + success: ProductEventsFunnelResponse, + // A funnel the builder rejects (no steps, >10, session step past step 1, + // non-positive window) is a 400, not a warehouse failure. + error: validatedQueryEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("productEventsFunnelBreakdown", "/product-events-funnel-breakdown", { + payload: ProductEventsFunnelBreakdownRequest, + success: ProductEventsFunnelBreakdownResponse, + error: validatedQueryEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("productEventNames", "/product-event-names", { + payload: ProductEventNamesRequest, + success: ProductEventNamesResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("executeRawSql", "/execute-raw-sql", { payload: RawSqlExecuteRequest, diff --git a/packages/domain/src/http/v2/dashboard-widget-parity.test.ts b/packages/domain/src/http/v2/dashboard-widget-parity.test.ts index 3d13bde6d..b69ad913a 100644 --- a/packages/domain/src/http/v2/dashboard-widget-parity.test.ts +++ b/packages/domain/src/http/v2/dashboard-widget-parity.test.ts @@ -100,7 +100,17 @@ const fullDisplay = { listLimit: 50, listRootOnly: true, pie: { donut: true, innerRadius: 2, showLabels: true, showPercent: true }, - funnel: { showStepPercent: true }, + funnel: { + showStepPercent: true, + steps: [ + { kind: "session", dimension: "utmSource", value: "twitter" }, + { kind: "page", pagePath: "/pricing", host: "example.com" }, + { kind: "event", eventName: "signup_completed", attributeEquals: { plan: "pro" } }, + ], + keyBy: "person", + windowSeconds: 86400, + breakdownBy: "attribute:plan", + }, histogram: { bucketCount: 10, bucketWidth: 2, logScaleY: false }, heatmap: { colorScale: "amber", scaleType: "linear" }, gauge: { min: 0, max: 100, style: "radial" }, @@ -139,7 +149,18 @@ describe("the v2 widget display mirrors the stored one", () => { pie: { inner_radius: 2, show_labels: true, show_percent: true }, histogram: { bucket_count: 10, bucket_width: 2, log_scale_y: false }, heatmap: { color_scale: "amber", scale_type: "linear" }, - funnel: { show_step_percent: true }, + funnel: { + show_step_percent: true, + key_by: "person", + window_seconds: 86400, + breakdown_by: "attribute:plan", + // Steps are the query-engine contract and stay camelCase inside. + steps: [ + { kind: "session", dimension: "utmSource", value: "twitter" }, + { kind: "page", pagePath: "/pricing", host: "example.com" }, + { kind: "event", eventName: "signup_completed", attributeEquals: { plan: "pro" } }, + ], + }, sparkline: { data_source: { kind: "route", endpoint: "list_traces" } }, list_where_clause: "service.name = $service", list_root_only: true, diff --git a/packages/domain/src/http/v2/dashboards.ts b/packages/domain/src/http/v2/dashboards.ts index 0091c8e95..99ed533a0 100644 --- a/packages/domain/src/http/v2/dashboards.ts +++ b/packages/domain/src/http/v2/dashboards.ts @@ -33,6 +33,9 @@ import { } from "../share" import { SORT_DIRECTIONS, STAT_AGGREGATES } from "@maple/widgets/dashboard" import { + FunnelBreakdownBy, + FunnelKeyBy, + FunnelStep, QUERY_RESULT_KINDS, QueryBuilderFormulaSchema, QueryBuilderQueryDraftSchema, @@ -332,9 +335,24 @@ export const V2WidgetDisplay = Schema.Struct({ }), ), ), + // The product-event funnel definition rides on the display block (see the + // stored schema in `@maple/widgets`). Step objects keep their own camelCase + // keys on the wire: they are the query-engine's `FunnelStep` contract, the + // same shape `query_funnel` and the internal endpoint speak. funnel: optional( - Schema.Struct({ showStepPercent: optional(Schema.Boolean) }).pipe( - Schema.encodeKeys({ showStepPercent: "show_step_percent" }), + Schema.Struct({ + showStepPercent: optional(Schema.Boolean), + steps: optional(Schema.Array(FunnelStep)), + keyBy: optional(FunnelKeyBy), + windowSeconds: optional(Schema.Number), + breakdownBy: optional(FunnelBreakdownBy), + }).pipe( + Schema.encodeKeys({ + showStepPercent: "show_step_percent", + keyBy: "key_by", + windowSeconds: "window_seconds", + breakdownBy: "breakdown_by", + }), ), ), histogram: optional( diff --git a/packages/domain/src/mcp-structured-types.ts b/packages/domain/src/mcp-structured-types.ts index 7faa10615..037f931a7 100644 --- a/packages/domain/src/mcp-structured-types.ts +++ b/packages/domain/src/mcp-structured-types.ts @@ -964,3 +964,44 @@ export type StructuredToolOutput = tool: "update_error_notification_policy" data: UpdateErrorNotificationPolicyData } + | { tool: "query_funnel"; data: QueryFunnelData } + | { tool: "list_product_events"; data: ListProductEventsData } + +// Product-event funnels + +export interface QueryFunnelStepData { + /** 1-based. */ + step: number + label: string + count: number + /** Share of step 1, 0–1. */ + ofFirst: number + /** Conversion from the previous step, 0–1; null on step 1 or when the previous step counted nobody. */ + ofPrevious: number | null + dropOff: number +} + +export interface QueryFunnelData { + timeRange: { start: string; end: string } + keyBy: "person" | "visitor" | "user" | "session" + windowSeconds: number + steps: ReadonlyArray + /** Last step over first, 0–1; null with fewer than two steps or an empty first step. */ + conversion: number | null + breakdown?: { + by: string + groups: ReadonlyArray<{ group: string; counts: ReadonlyArray; conversion: number | null }> + } +} + +export interface ListProductEventsData { + timeRange: { start: string; end: string } + events: ReadonlyArray<{ + eventName: string + /** `navigation` (page view), `custom` (`track()`), `screen` (mobile). */ + kind: string + count: number + sessions: number + persons: number + }> +} diff --git a/packages/query-engine/src/observability/index.ts b/packages/query-engine/src/observability/index.ts index d7df4dfe7..235b8c8da 100644 --- a/packages/query-engine/src/observability/index.ts +++ b/packages/query-engine/src/observability/index.ts @@ -45,3 +45,14 @@ export { type SessionReplayDetailOutput, type SessionTraceSummaryOutput, } from "./session-replays" +export { + productEventsFunnel, + productEventsFunnelBreakdown, + productEventNames, + type ProductEventsFunnelInput, + type ProductEventsFunnelBreakdownInput, + type ProductEventNamesInput, + type ProductEventsFunnelOutput, + type ProductEventsFunnelBreakdownOutput, + type ProductEventNamesOutput, +} from "./product-events" diff --git a/packages/query-engine/src/observability/product-events.ts b/packages/query-engine/src/observability/product-events.ts new file mode 100644 index 000000000..269fd98cc --- /dev/null +++ b/packages/query-engine/src/observability/product-events.ts @@ -0,0 +1,89 @@ +import { Effect } from "effect" +import * as CH from "../ch" +import { WarehouseExecutor } from "./WarehouseExecutor" + +export type { + ProductEventNamesOutput, + ProductEventsFunnelBreakdownOutput, + ProductEventsFunnelOutput, +} from "../ch/queries/product-events" + +// Product-event funnels for the MCP tools. Thin wrappers over the CH builders in +// `../ch/queries/product-events.ts`; the builders validate the definition +// synchronously and throw `ProductEventsFunnelError`, which these surface as a +// typed failure so a tool can print the reason instead of dying. + +export interface ProductEventsFunnelInput extends CH.ProductEventsFunnelOpts { + readonly startTime: string + readonly endTime: string +} + +export interface ProductEventsFunnelBreakdownInput extends CH.ProductEventsFunnelBreakdownOpts { + readonly startTime: string + readonly endTime: string +} + +export interface ProductEventNamesInput extends CH.ProductEventNamesOpts { + readonly startTime: string + readonly endTime: string +} + +const build = (make: () => A): Effect.Effect => + Effect.try({ + try: make, + catch: (error) => { + if (error instanceof CH.ProductEventsFunnelError) return error + throw error + }, + }) + +/** Run a funnel: exactly one `{ step, count }` row per step, in step order. */ +export const productEventsFunnel = Effect.fn("Observability.productEventsFunnel")(function* ( + input: ProductEventsFunnelInput, +) { + const executor = yield* WarehouseExecutor + yield* Effect.annotateCurrentSpan({ + orgId: executor.orgId, + "funnel.steps": input.steps.length, + "funnel.keyBy": input.keyBy, + }) + const { startTime, endTime, ...opts } = input + const query = yield* build(() => CH.productEventsFunnelQuery(opts)) + const compiled = CH.compile(query, { orgId: executor.orgId, startTime, endTime }) + return yield* executor.compiledQuery(compiled, { profile: "aggregation", context: "productEventsFunnel" }) +}) + +/** Run a funnel broken down by a session dimension or an event attribute: `{ group, step, count }` rows. */ +export const productEventsFunnelBreakdown = Effect.fn("Observability.productEventsFunnelBreakdown")( + function* (input: ProductEventsFunnelBreakdownInput) { + const executor = yield* WarehouseExecutor + yield* Effect.annotateCurrentSpan({ + orgId: executor.orgId, + "funnel.steps": input.steps.length, + "funnel.keyBy": input.keyBy, + "funnel.breakdownBy": input.breakdownBy, + }) + const { startTime, endTime, ...opts } = input + const query = yield* build(() => CH.productEventsFunnelBreakdownQuery(opts)) + const compiled = CH.compile(query, { orgId: executor.orgId, startTime, endTime }) + return yield* executor.compiledQuery(compiled, { + profile: "aggregation", + context: "productEventsFunnelBreakdown", + }) + }, +) + +/** The event names in range, most frequent first: `{ eventName, kind, count, sessions, persons }`. */ +export const productEventNames = Effect.fn("Observability.productEventNames")(function* ( + input: ProductEventNamesInput, +) { + const executor = yield* WarehouseExecutor + yield* Effect.annotateCurrentSpan("orgId", executor.orgId) + const { startTime, endTime, ...opts } = input + const compiled = CH.compile(CH.productEventNamesQuery(opts), { + orgId: executor.orgId, + startTime, + endTime, + }) + return yield* executor.compiledQuery(compiled, { profile: "aggregation", context: "productEventNames" }) +}) diff --git a/packages/query-engine/src/registry/index.ts b/packages/query-engine/src/registry/index.ts index 1d1232ac2..675922d04 100644 --- a/packages/query-engine/src/registry/index.ts +++ b/packages/query-engine/src/registry/index.ts @@ -10,4 +10,5 @@ export { type TimeBucketQueryCachePolicy, } from "./query-definition" export * from "./logs" +export { productEventsFunnelOpts } from "./product-events" export * as Queries from "./queries" diff --git a/packages/query-engine/src/registry/product-events.ts b/packages/query-engine/src/registry/product-events.ts new file mode 100644 index 000000000..c11545d34 --- /dev/null +++ b/packages/query-engine/src/registry/product-events.ts @@ -0,0 +1,83 @@ +import type { + ProductEventNamesRequest, + ProductEventsFunnelBreakdownRequest, + ProductEventsFunnelRequest, +} from "@maple/domain/http" +import * as CH from "../ch" +import { timeRangeCache } from "../runtime/query-engine" +import { defineQuery } from "./query-definition" + +// Product-event funnels read `product_events` only — server and mobile rows have +// no raw `session_events` counterpart, so unlike the web-analytics pairs there is +// no `Raw` twin to fall back to. The builders validate the funnel definition +// synchronously and throw `ProductEventsFunnelError`; the HTTP handler checks the +// definition through `productEventsFunnelOpts` before `compile` runs here so a +// bad definition is a 400 rather than a defect. + +/** The web-analytics filter surface, read off any funnel-family request. */ +const productEventsFilters = (payload: ProductEventNamesRequest): CH.ProductEventsFilters => ({ + host: payload.host, + pagePath: payload.pagePath, + referrerHost: payload.referrerHost, + country: payload.country, + deviceType: payload.deviceType, + browserName: payload.browserName, + osName: payload.osName, + language: payload.language, + utmSource: payload.utmSource, + utmMedium: payload.utmMedium, + utmCampaign: payload.utmCampaign, + visitorType: payload.visitorType, + useProductEvents: true, +}) + +/** The funnel option bag shared by the plain and breakdown queries. */ +export const productEventsFunnelOpts = ( + payload: ProductEventsFunnelRequest | ProductEventsFunnelBreakdownRequest, +): CH.ProductEventsFunnelOpts => ({ + steps: payload.steps, + keyBy: payload.keyBy, + windowSeconds: payload.windowSeconds, + filters: productEventsFilters(payload), +}) + +export const productEventsFunnel = defineQuery({ + id: "productEventsFunnel", + profile: "aggregation", + cache: timeRangeCache, + compile: (payload: ProductEventsFunnelRequest, orgId: string) => + CH.compile(CH.productEventsFunnelQuery(productEventsFunnelOpts(payload)), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + +export const productEventsFunnelBreakdown = defineQuery({ + id: "productEventsFunnelBreakdown", + profile: "aggregation", + cache: timeRangeCache, + compile: (payload: ProductEventsFunnelBreakdownRequest, orgId: string) => + CH.compile( + CH.productEventsFunnelBreakdownQuery({ + ...productEventsFunnelOpts(payload), + breakdownBy: payload.breakdownBy, + limit: payload.limit, + }), + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + +export const productEventNames = defineQuery({ + id: "productEventNames", + profile: "aggregation", + cache: timeRangeCache, + compile: (payload: ProductEventNamesRequest, orgId: string) => + CH.compile( + CH.productEventNamesQuery({ + filters: productEventsFilters(payload), + limit: payload.limit, + }), + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index de694d943..e262d3116 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -46,6 +46,7 @@ import { makeTimeRangeCachePolicy, timeRangeCache } from "../runtime/query-engin import { defineQuery } from "./query-definition" export { logsCount, logsTimeseries } from "./logs" +export { productEventsFunnel, productEventsFunnelBreakdown, productEventNames } from "./product-events" /** * Declarative compile, execution, and cache policy. Handlers retain response diff --git a/packages/query-model/src/funnel.ts b/packages/query-model/src/funnel.ts new file mode 100644 index 000000000..d859499eb --- /dev/null +++ b/packages/query-model/src/funnel.ts @@ -0,0 +1,83 @@ +import { Schema } from "effect" + +// A product-event funnel definition, as every surface that stores one keeps +// it: the /analytics URL, a dashboard funnel widget, an MCP tool call, and the +// internal query-engine request. Mirrors the option types of +// `productEventsFunnelQuery` in `@maple/query-engine` field for field; the +// semantics (person stitching, the session step, the breakdown grouping) are +// documented there. + +/** Which `session_replays` dimension a `session` step (or a breakdown) reads. */ +export const FunnelSessionDimension = Schema.Literals([ + "referrerHost", + "utmSource", + "utmMedium", + "utmCampaign", + "country", + "host", +]) +export type FunnelSessionDimension = typeof FunnelSessionDimension.Type + +/** + * What a funnel counts: a stitched person (user id, else the visitor's linked + * user, else the visitor), the raw visitor or user column, or the session. + */ +export const FunnelKeyBy = Schema.Literals(["person", "visitor", "user", "session"]) +export type FunnelKeyBy = typeof FunnelKeyBy.Type + +/** A `track()` (or direct-ingested) event by name, optionally narrowed by `Attributes[k] = v`. */ +export const FunnelEventStep = Schema.Struct({ + kind: Schema.Literal("event"), + eventName: Schema.String, + attributeEquals: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), +}) +/** A page view of `pagePath`, optionally on one `host`. */ +export const FunnelPageStep = Schema.Struct({ + kind: Schema.Literal("page"), + pagePath: Schema.String, + host: Schema.optionalKey(Schema.String), +}) +/** "Started a session with this acquisition dimension" — only valid as step 1. */ +export const FunnelSessionStep = Schema.Struct({ + kind: Schema.Literal("session"), + dimension: FunnelSessionDimension, + value: Schema.String, +}) +export const FunnelStep = Schema.Union([FunnelEventStep, FunnelPageStep, FunnelSessionStep]) +export type FunnelStep = typeof FunnelStep.Type + +/** An acquisition dimension of the person's sessions, the event `Host`, or `attribute:`. */ +export const FunnelBreakdownBy = Schema.Union([ + FunnelSessionDimension, + Schema.TemplateLiteral(["attribute:", Schema.String]), +]) +export type FunnelBreakdownBy = typeof FunnelBreakdownBy.Type + +/** The most steps a funnel takes; mirrors `FUNNEL_MAX_STEPS` in the query engine. */ +export const FUNNEL_MAX_STEPS = 10 + +/** Sentence-case labels for the session dimensions, shared by every surface that names a step. */ +export const FUNNEL_SESSION_DIMENSION_LABEL = { + referrerHost: "Referrer", + utmSource: "UTM source", + utmMedium: "UTM medium", + utmCampaign: "UTM campaign", + country: "Country", + host: "Site", +} satisfies Record + +/** + * The human label for a step — the bar label in a funnel chart, the row label + * in a step table. One function so the /analytics view, the dashboard widget + * (browser and share API alike) and the MCP tools print the same thing. + */ +export const funnelStepLabel = (step: FunnelStep): string => { + switch (step.kind) { + case "event": + return step.eventName + case "page": + return step.host ? `${step.host}${step.pagePath}` : step.pagePath + case "session": + return `${FUNNEL_SESSION_DIMENSION_LABEL[step.dimension]}: ${step.value}` + } +} diff --git a/packages/query-model/src/index.ts b/packages/query-model/src/index.ts index 753ddfccf..ab08cc0ff 100644 --- a/packages/query-model/src/index.ts +++ b/packages/query-model/src/index.ts @@ -19,6 +19,7 @@ export * from "./comparison" export * from "./formula" +export * from "./funnel" export * from "./query-draft" export * from "./query-set" export * from "./result-shape" diff --git a/packages/widgets/src/dashboard/construct.test.ts b/packages/widgets/src/dashboard/construct.test.ts index d34555dc9..a4938f22c 100644 --- a/packages/widgets/src/dashboard/construct.test.ts +++ b/packages/widgets/src/dashboard/construct.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest" -import { dataSourceEndpoint, dataSourceQuerySet, dataSourceRawSql } from "./access" -import { makeQueryDataSource, makeRawSqlDataSource, makeRouteDataSource } from "./construct" +import { dataSourceEndpoint, dataSourceQuerySet, dataSourceRawSql, dataSourceRouteParams } from "./access" +import { + makeProductEventsFunnelDataSource, + makeQueryDataSource, + makeRawSqlDataSource, + makeRouteDataSource, + PRODUCT_EVENTS_FUNNEL_ENDPOINT, +} from "./construct" /** * The round trip is the whole contract: whatever a constructor writes, the @@ -103,3 +109,22 @@ describe("makeRouteDataSource", () => { }) }) }) + +describe("makeProductEventsFunnelDataSource", () => { + const steps = [ + { kind: "page" as const, pagePath: "/pricing" }, + { kind: "event" as const, eventName: "signup_completed" }, + ] + + it("is a route the accessors read back, with the definition as its params", () => { + const source = makeProductEventsFunnelDataSource({ steps, keyBy: "person", windowSeconds: 3600 }) + expect(dataSourceEndpoint(source)).toBe(PRODUCT_EVENTS_FUNNEL_ENDPOINT) + expect(dataSourceRouteParams(source)).toEqual({ steps, keyBy: "person", windowSeconds: 3600 }) + expect(dataSourceQuerySet(source)).toBeNull() + expect(dataSourceRawSql(source)).toBeNull() + }) + + it("forwards only the definition fields that are set", () => { + expect(dataSourceRouteParams(makeProductEventsFunnelDataSource({ steps }))).toEqual({ steps }) + }) +}) diff --git a/packages/widgets/src/dashboard/construct.ts b/packages/widgets/src/dashboard/construct.ts index 0b0d84fa4..cdb947af2 100644 --- a/packages/widgets/src/dashboard/construct.ts +++ b/packages/widgets/src/dashboard/construct.ts @@ -1,4 +1,4 @@ -import type { QueryResultContract, QuerySet } from "@maple/query-model" +import type { FunnelKeyBy, FunnelStep, QueryResultContract, QuerySet } from "@maple/query-model" import type { RawSqlDataSource } from "./access" import type { WidgetDataSourceTransformV2 } from "./shared/transform" import type { @@ -128,3 +128,36 @@ export const makeStaticDataSource = ( kind: "static", ...(!(transform === undefined) ? { transform } : undefined), }) + +/** The route a product-event funnel widget fetches through. */ +export const PRODUCT_EVENTS_FUNNEL_ENDPOINT = "product_events_funnel" + +/** + * The stored `display.funnel` definition of a product-event funnel widget — + * what `makeProductEventsFunnelDataSource` reads. + */ +export interface ProductEventsFunnelDefinition { + readonly steps: ReadonlyArray + readonly keyBy?: FunnelKeyBy + readonly windowSeconds?: number +} + +/** + * A funnel widget over `product_events`: the definition mirrored into the route + * params, so the fetch path (`toWidgetRequest`) never has to read the display. + * `keyBy` and `windowSeconds` are forwarded only when set; the route applies + * the same defaults the /analytics Funnels view does. + */ +export const makeProductEventsFunnelDataSource = ( + funnel: ProductEventsFunnelDefinition, + transform?: WidgetDataSourceTransform, +) => + makeRouteDataSource( + PRODUCT_EVENTS_FUNNEL_ENDPOINT, + { + steps: funnel.steps, + ...(!(funnel.keyBy === undefined) ? { keyBy: funnel.keyBy } : undefined), + ...(!(funnel.windowSeconds === undefined) ? { windowSeconds: funnel.windowSeconds } : undefined), + }, + transform, + ) diff --git a/packages/widgets/src/dashboard/index.ts b/packages/widgets/src/dashboard/index.ts index 67aa5ed63..ab4bf500c 100644 --- a/packages/widgets/src/dashboard/index.ts +++ b/packages/widgets/src/dashboard/index.ts @@ -39,10 +39,13 @@ export { } from "./access" export { toWidgetRequest, type WidgetRequest } from "./request" export { + makeProductEventsFunnelDataSource, makeQueryDataSource, makeRawSqlDataSource, makeRouteDataSource, makeStaticDataSource, + PRODUCT_EVENTS_FUNNEL_ENDPOINT, + type ProductEventsFunnelDefinition, type QueryDataSourceInput, type RawSqlDataSourceInput, } from "./construct" diff --git a/packages/widgets/src/dashboard/shared/display.ts b/packages/widgets/src/dashboard/shared/display.ts index 0c31e9542..7b288a0d9 100644 --- a/packages/widgets/src/dashboard/shared/display.ts +++ b/packages/widgets/src/dashboard/shared/display.ts @@ -1,3 +1,4 @@ +import { FunnelBreakdownBy, FunnelKeyBy, FunnelStep } from "@maple/query-model" import { Schema } from "effect" import { HEATMAP_COLOR_SCALES, HEATMAP_SCALE_TYPES } from "../../widget-types" import { StringRecord } from "./transform" @@ -110,10 +111,21 @@ export const makeWidgetDisplayConfigSchema = (dat }), ), - // Funnel-specific + // Funnel-specific. + // + // `steps`/`keyBy`/`windowSeconds`/`breakdownBy` turn the funnel from a + // rendering of group-by rows into a product-event funnel: when `steps` is + // present the widget is fetched through the funnel endpoint instead of the + // query set. Additive — a funnel without them renders exactly as before, + // which is what keeps older readers of the document (the mobile app reads + // this wire) safe. funnel: Schema.optional( Schema.Struct({ showStepPercent: Schema.optional(Schema.Boolean), + steps: Schema.optional(Schema.Array(FunnelStep)), + keyBy: Schema.optional(FunnelKeyBy), + windowSeconds: Schema.optional(Schema.Number), + breakdownBy: Schema.optional(FunnelBreakdownBy), }), ), From a1b5d0581563b9f7660bdcbbca70e4306fa1bd9d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 13:13:44 +0200 Subject: [PATCH 09/15] chore: lint/format fixes on touched files; console warnings out of Effect code in MapleEvents --- apps/api/src/mcp/tools/registry.ts | 4 +- .../src/services/product-events/svix.test.ts | 10 +-- ...eb-analytics-parity.clickhouse.e2e.test.ts | 23 +++++- .../src/content/docs/sdks/effect-server.md | 6 +- .../docs/session-replay/product-events-api.md | 36 ++++----- docs/product-events-funnels.md | 32 +++++--- .../src/ch/core-dsl.test.ts | 8 +- .../src/clickhouse/migrations/index.test.ts | 4 +- packages/effect-sdk/src/server/events.ts | 27 ++++--- .../query-engine/src/ch/builder-fixtures.ts | 17 +++- .../src/ch/queries/product-events.test.ts | 81 +++++++++++++++---- .../src/ch/queries/product-events.ts | 49 +++++++---- 12 files changed, 206 insertions(+), 91 deletions(-) diff --git a/apps/api/src/mcp/tools/registry.ts b/apps/api/src/mcp/tools/registry.ts index 72c3379b6..85607a931 100644 --- a/apps/api/src/mcp/tools/registry.ts +++ b/apps/api/src/mcp/tools/registry.ts @@ -131,9 +131,7 @@ const collapseNullableUnions = (node: unknown): unknown => { return collapseNullableUnions({ ...kept, ...siblings }) } } - return Object.fromEntries( - Object.entries(obj).map(([key, value]) => [key, collapseNullableUnions(value)]), - ) + return Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, collapseNullableUnions(value)])) } export const toInputSchema = (schema: Schema.Top): Record => { diff --git a/apps/api/src/services/product-events/svix.test.ts b/apps/api/src/services/product-events/svix.test.ts index d8c5a4db9..3ea8fdf13 100644 --- a/apps/api/src/services/product-events/svix.test.ts +++ b/apps/api/src/services/product-events/svix.test.ts @@ -29,16 +29,10 @@ describe("svix", () => { }), ) - it.effect("accepts a valid delivery", () => - Effect.gen(function* () { - yield* verify() - }), - ) + it.effect("accepts a valid delivery", () => verify()) it.effect("accepts when the matching signature is one of several (secret rotation)", () => - Effect.gen(function* () { - yield* verify({ headers: headers({ "svix-signature": `v1,AAAA= ${SIGNATURE} v2,ignored` }) }) - }), + verify({ headers: headers({ "svix-signature": `v1,AAAA= ${SIGNATURE} v2,ignored` }) }), ) it.effect("rejects a tampered body", () => diff --git a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts index 1457d989c..5dbe179ec 100644 --- a/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts @@ -376,7 +376,9 @@ describe.skipIf(!clickhouseE2eEnabled)("web analytics raw-vs-rollup parity", () }) it("populates product_events from the materialized view with only navigation and custom rows", async () => { - const rows = await runJson("SELECT Kind, count() AS n FROM product_events GROUP BY Kind ORDER BY Kind") + const rows = await runJson( + "SELECT Kind, count() AS n FROM product_events GROUP BY Kind ORDER BY Kind", + ) const byKind = Object.fromEntries(rows.map((row) => [String(row.Kind), Number(row.n)])) const expectedNavigation = SEED_EVENTS.filter((row) => row.type === "navigation").length const expectedCustom = SEED_EVENTS.filter((row) => row.type === "custom").length @@ -463,7 +465,16 @@ const fev = ( const FUNNEL_EVENTS: ReadonlyArray = [ fev("f1", "v1", "", at(HOUR_MS), 0, "navigation", "https://maple.dev/"), fev("f1", "v1", "", at(HOUR_MS + MINUTE_MS), 1, "navigation", "https://maple.dev/pricing"), - fev("f1", "v1", "", at(HOUR_MS + 2 * MINUTE_MS), 2, "custom", "https://maple.dev/pricing", "signup_started"), + fev( + "f1", + "v1", + "", + at(HOUR_MS + 2 * MINUTE_MS), + 2, + "custom", + "https://maple.dev/pricing", + "signup_started", + ), fev("f2", "v2", "", at(2 * HOUR_MS), 0, "navigation", "https://maple.dev/"), fev("f2", "v2", "", at(2 * HOUR_MS + MINUTE_MS), 1, "navigation", "https://maple.dev/pricing"), fev("f3", "v3", "", at(3 * HOUR_MS), 0, "navigation", "https://maple.dev/pricing"), @@ -640,7 +651,13 @@ describe.skipIf(!clickhouseE2eEnabled)("product events funnels", () => { it("lists event names with counts, sessions and persons", async () => { const rows = await runJson(CH.compile(CH.productEventNamesQuery({ limit: 10 }), funnelWindow).sql) assert.deepStrictEqual( - rows.map((row) => [row.eventName, row.kind, Number(row.count), Number(row.sessions), Number(row.persons)]), + rows.map((row) => [ + row.eventName, + row.kind, + Number(row.count), + Number(row.sessions), + Number(row.persons), + ]), [ ["$pageview", "navigation", 6, 4, 4], ["plan_started", "custom", 2, 0, 2], diff --git a/apps/landing/src/content/docs/sdks/effect-server.md b/apps/landing/src/content/docs/sdks/effect-server.md index 5b73d07e1..44b80d572 100644 --- a/apps/landing/src/content/docs/sdks/effect-server.md +++ b/apps/landing/src/content/docs/sdks/effect-server.md @@ -97,7 +97,11 @@ import { Effect, Layer } from "effect" const EventsLive = MapleEvents.layer({ serviceName: "billing" }) -const onSubscriptionCreated = Effect.fn("onSubscriptionCreated")(function* (userId: string, orgId: string, plan: string) { +const onSubscriptionCreated = Effect.fn("onSubscriptionCreated")(function* ( + userId: string, + orgId: string, + plan: string, +) { const events = yield* MapleEvents.MapleEvents yield* events.track("plan_started", { userId, groupId: orgId, attributes: { plan } }) }) diff --git a/apps/landing/src/content/docs/session-replay/product-events-api.md b/apps/landing/src/content/docs/session-replay/product-events-api.md index b09d76f01..f24a6eb0d 100644 --- a/apps/landing/src/content/docs/session-replay/product-events-api.md +++ b/apps/landing/src/content/docs/session-replay/product-events-api.md @@ -30,31 +30,31 @@ resolved from the ingest key — an `org_id` in the body is ignored. ## Fields -| Field | Type | Notes | -| -------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Field | Type | Notes | +| -------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `name` | string | **Required.** 1–128 bytes. Names starting with `$` are reserved for Maple's SDKs and dropped, except `$screen` (mobile screen view, stored as `Kind = screen`). | -| `timestamp` | string | RFC 3339 (`2026-08-17T10:15:30.123Z`) or `YYYY-MM-DD HH:MM:SS[.fff]` (UTC). Defaults to the time the gateway received the batch. Stored as UTC. | -| `source` | string | `server` (default) or `mobile`. `browser` is reserved for the SDKs; other values drop the row. | -| `visitor_id` | string | Anonymous/device id — the browser SDK cookie value or a persistent mobile install id. ≤ 256 bytes. | -| `user_id` | string | Your user id after sign-in, matching what you pass to `identify()`. ≤ 256 bytes. | -| `group_id` | string | Account / workspace / org id. ≤ 256 bytes. | -| `session_id` | string | Optional link to a browser or mobile session. ≤ 256 bytes. | -| `service_name` | string | The emitting service (`maple-api`, `acme-ios`). ≤ 128 bytes. | -| `url` | string | Optional. `host` (lowercase) and `page_path` (pathname only) are derived from it. | -| `page_path` | string | Optional explicit path; overrides the one derived from `url`. Mobile `$screen` events put the screen name here. | -| `attributes` | object | Optional properties. ≤ 32 keys, key ≤ 64 bytes, value ≤ 1024 bytes; non-string values are stringified. | +| `timestamp` | string | RFC 3339 (`2026-08-17T10:15:30.123Z`) or `YYYY-MM-DD HH:MM:SS[.fff]` (UTC). Defaults to the time the gateway received the batch. Stored as UTC. | +| `source` | string | `server` (default) or `mobile`. `browser` is reserved for the SDKs; other values drop the row. | +| `visitor_id` | string | Anonymous/device id — the browser SDK cookie value or a persistent mobile install id. ≤ 256 bytes. | +| `user_id` | string | Your user id after sign-in, matching what you pass to `identify()`. ≤ 256 bytes. | +| `group_id` | string | Account / workspace / org id. ≤ 256 bytes. | +| `session_id` | string | Optional link to a browser or mobile session. ≤ 256 bytes. | +| `service_name` | string | The emitting service (`maple-api`, `acme-ios`). ≤ 128 bytes. | +| `url` | string | Optional. `host` (lowercase) and `page_path` (pathname only) are derived from it. | +| `page_path` | string | Optional explicit path; overrides the one derived from `url`. Mobile `$screen` events put the screen name here. | +| `attributes` | object | Optional properties. ≤ 32 keys, key ≤ 64 bytes, value ≤ 1024 bytes; non-string values are stringified. | Over-long strings are truncated at the caps above; unknown fields are discarded. ## Responses -| Status | Meaning | -| ------ | ----------------------------------------------------------------------------------------------------------- | +| Status | Meaning | +| ------ | --------------------------------------------------------------------------------------------------------------------------------------- | | `200` | `{"accepted": }` — rows durably queued. Malformed rows (bad `name`, `source`, `timestamp`) are dropped individually and not counted. | -| `400` | A line is not valid JSON, or not a JSON object. The whole batch is rejected. | -| `401` | Missing or invalid ingest key. | -| `402` | The organization is out of quota for browser sessions (product events share that entitlement). | -| `503` | Storage temporarily unavailable — retry with backoff. | +| `400` | A line is not valid JSON, or not a JSON object. The whole batch is rejected. | +| `401` | Missing or invalid ingest key. | +| `402` | The organization is out of quota for browser sessions (product events share that entitlement). | +| `503` | Storage temporarily unavailable — retry with backoff. | Product events are not metered separately: they are covered by the browser-sessions entitlement. diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index 7c6f1aaad..2a483d121 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -66,11 +66,11 @@ Decisions baked in: `packages/browser-session/src/events/events-sink.ts` (`visitor_id`, `user_id`, `group_id` on each NDJSON line). The MV copies them through. No MV-side join against `session_replays` — that would depend on insert ordering. - - Requires adding `VisitorId`/`UserId`/`GroupId` (`DEFAULT ''`) to `session_events` too - (migration + Tinybird forward query, same shape as the 0012 `Attributes` widening). - - Rows from older SDKs arrive with `''`; the funnel falls back to a `session_replays` lookup for - those, or simply excludes them — pick "exclude + show coverage" (matches how the analytics page - already treats sessions without the 0011 block). + - Requires adding `VisitorId`/`UserId`/`GroupId` (`DEFAULT ''`) to `session_events` too + (migration + Tinybird forward query, same shape as the 0012 `Attributes` widening). + - Rows from older SDKs arrive with `''`; the funnel falls back to a `session_replays` lookup for + those, or simply excludes them — pick "exclude + show coverage" (matches how the analytics page + already treats sessions without the 0011 block). - **Sorting key**: `Timestamp` second so time ranges are a primary-index scan (the reason `web_events` exists), then `VisitorId` so a per-person `windowFunnel` groups over contiguous rows. Keep `SessionId`/`Seq` for stable step order. @@ -90,9 +90,17 @@ Ingest gateway (`apps/ingest/src/main.rs`), NDJSON like `/v1/sessionEvents`, aut ingest key (org from the key, never the body). Body per line: ```json -{ "timestamp": "...", "name": "plan_started", "source": "server", - "user_id": "user_…", "group_id": "org_…", "visitor_id": "", "session_id": "", - "service_name": "maple-api", "attributes": { "plan": "startup" } } +{ + "timestamp": "...", + "name": "plan_started", + "source": "server", + "user_id": "user_…", + "group_id": "org_…", + "visitor_id": "", + "session_id": "", + "service_name": "maple-api", + "attributes": { "plan": "startup" } +} ``` - Same caps as `sanitize_session_event` (name 128, ≤32 props, key 64, value 1024, 8 KiB total). @@ -102,7 +110,7 @@ ingest key (org from the key, never the body). Body per line: `clickhouse_insert_mappings.rs`, entitlement check reusing the browser-sessions feature id (or a new `product_events` feature — billing decision, default: reuse). - Writes go where session events go today (Tinybird managed; BYO CH via the export lane). -- Mobile: no SDK in scope. The endpoint *is* the contract; a mobile app posts with a persistent +- Mobile: no SDK in scope. The endpoint _is_ the contract; a mobile app posts with a persistent install id as `visitor_id` and `identify`-equivalent `user_id`. `@maple-dev/effect-sdk` server side gets `track()` (`packages/effect-sdk/src/server`?) as a thin client of this endpoint so Node/Bun backends have the same call as the browser. @@ -140,14 +148,16 @@ is one row per (visitor,user) pair. ### 5. Query engine `lib/clickhouse-builder`: + - Parametric aggregates `windowFunnel(windowSec, mode?)(ts, cond1..condN)` and `sequenceMatch(pattern)(ts, cond…)` following the handwritten `quantile(q)` pattern in `src/ch/functions/aggregate.ts:74`. `retention()` optional. `packages/query-engine/src/ch/queries/product-events.ts` (replaces `web-analytics.ts`'s `web_events` references; the page-view queries move here unchanged): + - `productEventsFunnelQuery({ steps, keyBy: "person" | "visitor" | "user" | "session", - windowSeconds, filters })` → per-step `count`, `conversion_from_prev`, `conversion_from_first`. +windowSeconds, filters })` → per-step `count`, `conversion_from_prev`, `conversion_from_first`. Step = `{ eventName } | { pagePath, host? } | { referrerHost | utmSource | ... }`. A session-dimension step (referral) becomes step 0's condition through the person's first `session_replays` row; event steps are `EventName = …` conditions on `product_events`. @@ -165,7 +175,7 @@ is one row per (visitor,user) pair. ### 6. Surfaces - **Dashboard widget**: new `funnel` config that is step-based (not group-by). Today's `funnel` - render shape stays as the renderer; the *widget type* gains `steps[]`, `keyBy`, `window`. + render shape stays as the renderer; the _widget type_ gains `steps[]`, `keyBy`, `window`. `packages/domain/src/http/v2/dashboards.ts` + parity test, `dashboard-schema-doc.ts` for MCP. - **`/analytics` → Funnels tab** reusing the existing filter sidebar (`WebAnalyticsFilters` become `ProductEventsFilters`), plus an event-name breakdown panel. diff --git a/lib/clickhouse-builder/src/ch/core-dsl.test.ts b/lib/clickhouse-builder/src/ch/core-dsl.test.ts index 54c2fa429..6e394bc2d 100644 --- a/lib/clickhouse-builder/src/ch/core-dsl.test.ts +++ b/lib/clickhouse-builder/src/ch/core-dsl.test.ts @@ -214,7 +214,9 @@ describe("parametric aggregates", () => { level: CH.windowFunnel(86400, "strict_order")($.Timestamp, $.Name.eq("a"), $.Name.eq("b")), })) const { sql } = compileCH(q, {}) - expect(sql).toContain("windowFunnel(86400, 'strict_order')(Timestamp, Name = 'a', Name = 'b') AS level") + expect(sql).toContain( + "windowFunnel(86400, 'strict_order')(Timestamp, Name = 'a', Name = 'b') AS level", + ) }) it("windowFunnel refuses an empty condition list", () => { @@ -227,7 +229,9 @@ describe("parametric aggregates", () => { matched: CH.sequenceMatch("(?1)(?t<3600)(?2)")($.Timestamp, $.Name.eq("a"), $.Name.eq("b")), })) const { sql } = compileCH(q, {}) - expect(sql).toContain("sequenceMatch('(?1)(?t<3600)(?2)')(Timestamp, Name = 'a', Name = 'b') AS matched") + expect(sql).toContain( + "sequenceMatch('(?1)(?t<3600)(?2)')(Timestamp, Name = 'a', Name = 'b') AS matched", + ) }) it("sequenceMatch refuses a pattern that could break out of the literal", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index a73db7e01..c28b679c7 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -36,7 +36,9 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) + expect(migrations.map((m) => m.version)).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]) expect(migrations.at(-1)).toBe(migration_0016_product_events) expect(latestMigrationVersion).toBe(16) // 0010, 0014 and 0015 are performance-only and skipped by the ingest-gating diff --git a/packages/effect-sdk/src/server/events.ts b/packages/effect-sdk/src/server/events.ts index dbc101f2c..4dc0d0132 100644 --- a/packages/effect-sdk/src/server/events.ts +++ b/packages/effect-sdk/src/server/events.ts @@ -161,6 +161,21 @@ export const make = Effect.fn("MapleEvents.make")(function* (config: MapleEvents lastPostWarnAt = now console.warn(`[MapleEvents] ${error.message} (dropping batch)`, error.cause ?? "") } + // Host-app-facing developer warnings, same posture as `warnIfDoomed` in + // layer.ts: these go to the console, not to the Effect logger, because the + // SDK's own logger may be exporting to the very endpoint that is misconfigured. + const warnMissingKey = (): void => { + if (warnedAboutKey) return + warnedAboutKey = true + console.warn( + "[MapleEvents] no ingest key — set MAPLE_INGEST_KEY or pass `ingestKey`; dropping events", + ) + } + const warnEmptyName = (): void => { + if (warnedAboutName) return + warnedAboutName = true + console.warn("[MapleEvents] track() needs a non-empty event name; the call was ignored.") + } const post = (lines: ReadonlyArray) => HttpClientRequest.post(url).pipe( @@ -193,12 +208,7 @@ export const make = Effect.fn("MapleEvents.make")(function* (config: MapleEvents const batch = buffer buffer = [] if (ingestKey === undefined) { - if (!warnedAboutKey) { - warnedAboutKey = true - console.warn( - "[MapleEvents] no ingest key — set MAPLE_INGEST_KEY or pass `ingestKey`; dropping events", - ) - } + warnMissingKey() return Effect.void } return post(batch).pipe( @@ -224,10 +234,7 @@ export const make = Effect.fn("MapleEvents.make")(function* (config: MapleEvents Effect.sync(() => { const trimmed = typeof name === "string" ? name.trim() : "" if (trimmed.length === 0) { - if (!warnedAboutName) { - warnedAboutName = true - console.warn("[MapleEvents] track() needs a non-empty event name; the call was ignored.") - } + warnEmptyName() return false } buffer.push({ diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index e3ab52a15..acd276a10 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -101,7 +101,10 @@ const webAnalyticsFixtures: ReadonlyArray = [ CH.compile(CH.webAnalyticsTimeseriesQuery({ bucketSeconds: 3600, useProductEvents }), window), ), ...webAnalyticsVariants("webAnalyticsPageviewsTimeseriesQuery", "default", (useProductEvents) => - CH.compile(CH.webAnalyticsPageviewsTimeseriesQuery({ bucketSeconds: 3600, useProductEvents }), window), + CH.compile( + CH.webAnalyticsPageviewsTimeseriesQuery({ bucketSeconds: 3600, useProductEvents }), + window, + ), ), // Forces the semi-join: `referrerHost` is a session_replays-only dimension, // so the page-view source has to narrow through a subquery to honour it. @@ -164,7 +167,11 @@ const productEventsFixtures: ReadonlyArray = [ label: "person", compile: () => CH.compile( - CH.productEventsFunnelQuery({ steps: FUNNEL_STEPS, keyBy: "person", windowSeconds: 7 * 86_400 }), + CH.productEventsFunnelQuery({ + steps: FUNNEL_STEPS, + keyBy: "person", + windowSeconds: 7 * 86_400, + }), window, ), }, @@ -189,7 +196,11 @@ const productEventsFixtures: ReadonlyArray = [ label: "visitor-session-step", compile: () => CH.compile( - CH.productEventsFunnelQuery({ steps: REFERRAL_STEPS, keyBy: "visitor", windowSeconds: 3_600 }), + CH.productEventsFunnelQuery({ + steps: REFERRAL_STEPS, + keyBy: "visitor", + windowSeconds: 3_600, + }), window, ), }, diff --git a/packages/query-engine/src/ch/queries/product-events.test.ts b/packages/query-engine/src/ch/queries/product-events.test.ts index 20c0ae550..831359948 100644 --- a/packages/query-engine/src/ch/queries/product-events.test.ts +++ b/packages/query-engine/src/ch/queries/product-events.test.ts @@ -39,7 +39,10 @@ describe("productEventsFunnelQuery", () => { }) it("emits one windowFunnel condition per step and one output row per step", () => { - const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), params) + const { sql } = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), + params, + ) // The window is passed in the timestamp's unit — epoch milliseconds — since // windowFunnel does not accept DateTime64. expect(sql).toContain("windowFunnel(3600000)(ts, s1 = 1, s2 = 1, s3 = 1) AS level") @@ -51,32 +54,51 @@ describe("productEventsFunnelQuery", () => { }) it("projects each step as a flag and only reads rows matching some step", () => { - const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), params) - expect(sql).toContain("toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s1") + const { sql } = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 3_600 }), + params, + ) + expect(sql).toContain( + "toUInt8(((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev')) AS s1", + ) expect(sql).toContain("toUInt8(EventName = 'signup_completed') AS s2") - expect(sql).toContain("toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s3") + expect(sql).toContain( + "toUInt8((EventName = 'plan_started' AND Attributes['plan'] = 'startup')) AS s3", + ) expect(oneLine(sql)).toContain( "AND ((((Kind = 'navigation' AND PagePath = '/pricing') AND Host = 'maple.dev') OR EventName = 'signup_completed') OR (EventName = 'plan_started' AND Attributes['plan'] = 'startup'))", ) }) it("keys by the raw column for visitor / user / session and drops empty keys", () => { - const visitor = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), params).sql + const visitor = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), + params, + ).sql expect(visitor).toContain("VisitorId AS key") expect(visitor).toContain("AND VisitorId != ''") expect(visitor).not.toContain("identity_links") - const user = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "user", windowSeconds: 60 }), params).sql + const user = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "user", windowSeconds: 60 }), + params, + ).sql expect(user).toContain("UserId AS key") expect(user).toContain("AND UserId != ''") - const session = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "session", windowSeconds: 60 }), params).sql + const session = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "session", windowSeconds: 60 }), + params, + ).sql expect(session).toContain("SessionId AS key") expect(session).toContain("AND SessionId != ''") }) it("stitches the person key through identity_links aggregated per visitor", () => { - const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), params) + const { sql } = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), + params, + ) expect(oneLine(sql)).toContain( "LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId FROM identity_links WHERE OrgId = 'org_1' GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId", ) @@ -87,7 +109,11 @@ describe("productEventsFunnelQuery", () => { it("turns a session step 1 into a UNION ALL branch of session_replays entries", () => { const { sql } = compileCH( - productEventsFunnelQuery({ steps: [REFERRAL, ...STEPS], keyBy: "visitor", windowSeconds: 86_400 }), + productEventsFunnelQuery({ + steps: [REFERRAL, ...STEPS], + keyBy: "visitor", + windowSeconds: 86_400, + }), params, ) expect(sql).toContain("UNION ALL") @@ -97,12 +123,17 @@ describe("productEventsFunnelQuery", () => { ) expect(sql).toContain("AND ReferrerHost = 'news.ycombinator.com'") // The events branch never satisfies the session step. - expect(oneLine(sql)).toContain("SELECT VisitorId AS key, toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, 0 AS s1,") + expect(oneLine(sql)).toContain( + "SELECT VisitorId AS key, toUInt64(toUnixTimestamp64Milli(Timestamp)) AS ts, 0 AS s1,", + ) expect(sql).toContain("windowFunnel(86400000)(ts, s1 = 1, s2 = 1, s3 = 1, s4 = 1) AS level") }) it("has no session_replays branch without a session step", () => { - const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), params) + const { sql } = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60 }), + params, + ) expect(sql).not.toContain("UNION ALL") expect(sql).not.toContain("session_replays") }) @@ -125,12 +156,17 @@ describe("productEventsFunnelQuery", () => { // …applies the replays dimension directly… expect(flat).toContain("AND s.Country = 'DE'") // …and the page filter through the navigation semi-join on product_events. - expect(flat).toContain("AND s.SessionId IN (SELECT SessionId AS sessionId FROM product_events WHERE OrgId = 'org_1'") + expect(flat).toContain( + "AND s.SessionId IN (SELECT SessionId AS sessionId FROM product_events WHERE OrgId = 'org_1'", + ) expect(flat).toContain("AND Kind = 'navigation' AND PagePath = '/' GROUP BY sessionId)") }) it("omits the population subquery when no filter is set", () => { - const { sql } = compileCH(productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), params) + const { sql } = compileCH( + productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), + params, + ) expect(sql).not.toContain(" IN (SELECT") }) @@ -174,7 +210,9 @@ describe("productEventsFunnelBreakdownQuery", () => { expect(flat).toContain("argMinIf(dim, ts, dim != '') AS group") expect(flat).toContain("countIf(level >= 1) AS entered") expect(flat).toContain("GROUP BY group ORDER BY entered DESC, group ASC LIMIT 5") - expect(flat).toContain("SELECT group AS group, arrayJoin([1, 2, 3]) AS step, arrayElement(counts, step) AS count") + expect(flat).toContain( + "SELECT group AS group, arrayJoin([1, 2, 3]) AS step, arrayElement(counts, step) AS count", + ) expect(flat).toContain("ORDER BY group ASC, step ASC") }) @@ -212,7 +250,12 @@ describe("productEventsFunnelBreakdownQuery", () => { it("uses the event Host without a join", () => { const { sql } = compileCH( - productEventsFunnelBreakdownQuery({ steps: STEPS, keyBy: "visitor", windowSeconds: 60, breakdownBy: "host" }), + productEventsFunnelBreakdownQuery({ + steps: STEPS, + keyBy: "visitor", + windowSeconds: 60, + breakdownBy: "host", + }), params, ) expect(sql).toContain("Host AS dim") @@ -249,11 +292,15 @@ describe("productEventNamesQuery", () => { it("applies host directly and other filters through the session semi-join", () => { const { sql } = compileCH( - productEventNamesQuery({ filters: { host: "maple.dev", referrerHost: "t.co", pagePath: "/pricing" } }), + productEventNamesQuery({ + filters: { host: "maple.dev", referrerHost: "t.co", pagePath: "/pricing" }, + }), params, ) const flat = oneLine(sql) - expect(flat).toContain("AND Host = 'maple.dev' AND SessionId IN (SELECT SessionId AS sessionId FROM session_replays") + expect(flat).toContain( + "AND Host = 'maple.dev' AND SessionId IN (SELECT SessionId AS sessionId FROM session_replays", + ) expect(flat).toContain("AND ReferrerHost = 't.co'") // pagePath narrows sessions through the navigation semi-join, not the events. expect(flat).toContain("AND Kind = 'navigation' AND PagePath = '/pricing' GROUP BY sessionId)") diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index 8789d57e0..3960dca0d 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -24,11 +24,7 @@ import type { CHQuery, ColumnAccessor, ColumnDefs, JoinedColumnAccessor } from " import { Schema } from "effect" import { ProductEvents, IdentityLinks, SessionReplays } from "../tables" import { CHNumber } from "../schema" -import { - replaysWhere, - needsSessionSemiJoin, - type ProductEventsFilters, -} from "./web-analytics" +import { replaysWhere, needsSessionSemiJoin, type ProductEventsFilters } from "./web-analytics" export type { ProductEventsFilters } from "./web-analytics" @@ -68,7 +64,13 @@ type FunnelBranch = CHQuery()( "@maple/query-engine/ProductEventsFunnelError", { - reason: Schema.Literals(["NoSteps", "TooManySteps", "SessionStepNotFirst", "InvalidWindow", "InvalidLimit"]), + reason: Schema.Literals([ + "NoSteps", + "TooManySteps", + "SessionStepNotFirst", + "InvalidWindow", + "InvalidLimit", + ]), message: Schema.String, }, ) {} @@ -385,7 +393,9 @@ function eventsBranch(plan: FunnelPlan): FunnelBranch { // aliases actually declared above are ever read. let base: OpenJoinQuery = from(ProductEvents, "e") if (keyBy === "person") { - base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (e, link) => e.VisitorId.eq(link.VisitorId)) + base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (e, link) => + e.VisitorId.eq(link.VisitorId), + ) } if (sessionDims) { base = base.leftJoinQuery(sessionDims, "sd", (e, sd) => e.SessionId.eq(sd.SessionId)) @@ -424,7 +434,9 @@ function eventsBranch(plan: FunnelPlan): FunnelBranch { $.Timestamp.lte(param.dateTime("endTime")), anyStep, key.neq(""), - hasPopulationFilter(filters) ? inSubquery(key, matchingPersonsSubquery(keyBy, filters)) : undefined, + hasPopulationFilter(filters) + ? inSubquery(key, matchingPersonsSubquery(keyBy, filters)) + : undefined, ] }) } @@ -444,7 +456,9 @@ function sessionEntryBranch(plan: FunnelPlan, step: Extract = from(SessionReplays, "s") if (keyBy === "person") { - base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (s, link) => s.VisitorId.eq(link.VisitorId)) + base = base.leftJoinQuery(identityLinksByVisitor(), LINK_ALIAS, (s, link) => + s.VisitorId.eq(link.VisitorId), + ) } return base @@ -455,7 +469,10 @@ function sessionEntryBranch(plan: FunnelPlan, step: Extract { const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) @@ -465,7 +482,9 @@ function sessionEntryBranch(plan: FunnelPlan, step: Extract(sessionEntryBranch(plan, plan.sessionStep), events), "funnel_events") + ? fromUnion( + unionAll(sessionEntryBranch(plan, plan.sessionStep), events), + "funnel_events", + ) : fromQuery(events, "funnel_events") return source @@ -640,4 +662,3 @@ export function productEventNamesQuery( .limit(limit) .format("JSON") } - From 2d6e827805512d11771946937c9b7b953f5f447a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 13:14:10 +0200 Subject: [PATCH 10/15] =?UTF-8?q?docs:=20product-events=20plan=20=E2=86=92?= =?UTF-8?q?=20implemented=20+=20rollout=20checklist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/product-events-funnels.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index 2a483d121..b66c59a6f 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -1,8 +1,27 @@ # Product events + funnels — plan -Status: **planned, not started** (2026-08-17). Goal: answer "referral → signed up → started a -plan" for any org (and for Maple itself), from browser, mobile and backend events, as a real -funnel in the product rather than a hand-written `run_sql`. +Status: **implemented in code (2026-08-17), pending rollout.** Goal: answer "referral → signed +up → started a plan" for any org (and for Maple itself), from browser, mobile and backend events, +as a real funnel in the product rather than a hand-written `run_sql`. + +## Rollout checklist (operator steps the code cannot do) + +1. **Tinybird**: `bun run --cwd apps/api tinybird:deploy` creates `product_events`, + `identity_links` and their MVs. Then populate both from their sources with an explicit `tb` + step (the SDK has no populate option — same caveat as 0014). Until populated, page views + read as zero on managed orgs. Old `web_events`/`web_events_mv` can be removed from the + workspace afterwards. +2. **BYO ClickHouse**: migration 0016 bumps `clickHouseSchemaVersion` to 16 (`requiredForIngest` + default), so BYO orgs' ingest routing is un-ready until they apply schema. Deliberate — the + gateway now writes the new `session_events` columns and `product_events` directly. +3. **Secrets** (api worker): `CLERK_WEBHOOK_SECRET`, `AUTUMN_WEBHOOK_SECRET` (both routes answer + 503 until set); optional `MAPLE_PRODUCT_EVENTS_INGEST_KEY` (defaults to `MAPLE_INGEST_KEY`). + Register `POST /webhooks/clerk` (event `user.created`) in Clerk and `POST /webhooks/autumn` + (`billing.updated`) in Autumn. +4. **Ingest**: `INGEST_TINYBIRD_DATASOURCE_PRODUCT_EVENTS` defaults to `product_events`; nothing + to set unless the datasource name differs. +5. Publish `@maple-dev/effect-sdk` / `@maple-dev/browser` so customers' events start carrying + identity; older builds keep writing (all new columns default). ## Where we are From 8b911d13b4cd28165f5a54368ea566b01742c14c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 13:51:56 +0200 Subject: [PATCH 11/15] feat(billing): product_events is its own metered feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autumn feature product_events (Startup: 1M included, $0.05/1,000 — placeholder price). Ingest gates /v1/events on it and meters direct rows plus browser track() rows; session transcripts stay gated on browser_sessions. Web/landing billing UI, spend model (rate normalised per billingUnits), daily spend series, and pricing page carry the new feature. Retention already 365d. --- apps/api/autumn.config.ts | 24 + .../src/services/billing/DailySpendService.ts | 16 + apps/api/src/services/billing/autumn-http.ts | 3 +- apps/ingest/src/autumn.rs | 6 +- apps/ingest/src/main.rs | 449 +++++++++++++++--- apps/landing/messages/en.json | 2 + apps/landing/messages/ja.json | 2 + apps/landing/messages/ko.json | 2 + .../landing/src/components/PricingTable.astro | 10 +- .../docs/session-replay/product-events-api.md | 2 +- apps/landing/src/lib/pricing-offer.ts | 63 ++- apps/landing/src/pages/pricing.md.ts | 8 +- .../src/components/icons/cursor-pointer.tsx | 27 ++ apps/web/src/components/icons/index.ts | 1 + .../settings/feature-usage-cards.tsx | 27 +- .../src/components/settings/plan-offer.tsx | 2 +- .../src/components/settings/pricing-cards.tsx | 13 +- .../src/components/settings/spend-chart.tsx | 7 +- .../web/src/lib/billing/cost-estimate.test.ts | 18 +- apps/web/src/lib/billing/cost-estimate.ts | 17 +- apps/web/src/lib/billing/spend.test.ts | 42 +- apps/web/src/lib/billing/spend.ts | 77 ++- .../src/lib/services/atoms/billing-atoms.ts | 7 +- docs/product-events-funnels.md | 28 +- packages/domain/src/http/billing.ts | 12 +- .../src/product/billing-usage.test.ts | 26 + .../src/product/billing-usage.ts | 40 +- .../src/product/index.ts | 2 + 28 files changed, 798 insertions(+), 135 deletions(-) create mode 100644 apps/web/src/components/icons/cursor-pointer.tsx diff --git a/apps/api/autumn.config.ts b/apps/api/autumn.config.ts index 596327ca1..41f2ffb18 100644 --- a/apps/api/autumn.config.ts +++ b/apps/api/autumn.config.ts @@ -29,6 +29,16 @@ export const browserSessions = feature({ consumable: true, }) +// Product events (`track()` calls, `/v1/events` rows) — unit is one event. +// Metered by the ingest gateway on both paths (see `PRODUCT_EVENTS_FEATURE_ID` +// in apps/ingest/src/main.rs). +export const productEvents = feature({ + id: "product_events", + name: "Product Events", + type: "metered", + consumable: true, +}) + export const aiInputTokens = feature({ id: "ai_input_tokens", name: "AI Input Tokens", @@ -97,6 +107,20 @@ export const startup = plan({ interval: "month", }, }, + { + featureId: "product_events", + included: 1_000_000, + // PLACEHOLDER price, chosen 2026-08-17: $0.05 per 1,000 events past the + // first 1M/month (PostHog-style order of magnitude). `amount` is the + // price per `billingUnits` in atmn, so 0.05 / 1000 = $0.00005 per event. + // To be confirmed before the plan is pushed to production. + price: { + amount: 0.05, + billingUnits: 1000, + billingMethod: "usage_based", + interval: "month", + }, + }, ], freeTrial: { durationLength: 14, diff --git a/apps/api/src/services/billing/DailySpendService.ts b/apps/api/src/services/billing/DailySpendService.ts index 84d092e85..1550e1a95 100644 --- a/apps/api/src/services/billing/DailySpendService.ts +++ b/apps/api/src/services/billing/DailySpendService.ts @@ -97,11 +97,26 @@ export class DailySpendService extends Context.Service() for (const row of sessionRows) { sessionsByDay.set(toUtcDateKey(parseWarehouseDateTime(row.day)), row.sessions) } + const eventsByDay = new Map() + for (const row of eventRows) { + eventsByDay.set(toUtcDateKey(parseWarehouseDateTime(row.day)), row.events) + } + const days: DailyVolume[] = [] const firstDay = startOfUtcDay(cycle.startMs) const lastDay = startOfUtcDay(cycle.endMs) @@ -115,6 +130,7 @@ export class DailySpendService extends Context.Service = { * so one deep transform reproduces all of them. * * Some fields carry keys that are DATA, not schema — feature ids like - * `browser_sessions` / `ai_input_tokens`, group labels, free-form metadata. + * `browser_sessions` / `product_events` / `ai_input_tokens`, group labels, + * free-form metadata. * Camelizing those would silently break every feature lookup, so they are held * back. The SDK's inbound Zod schemas draw the line in two distinct places, and * so do we (the field's OWN key is always renamed either way — the SDK remapped diff --git a/apps/ingest/src/autumn.rs b/apps/ingest/src/autumn.rs index af1957325..f81f3d358 100644 --- a/apps/ingest/src/autumn.rs +++ b/apps/ingest/src/autumn.rs @@ -18,7 +18,8 @@ pub struct UsageEvent { pub org_id: String, pub feature_id: &'static str, /// Quantity to bill for this event. Unit depends on `feature_id`: GB for - /// `logs`/`traces`/`metrics`, a raw count for `browser_sessions`. + /// `logs`/`traces`/`metrics`, a raw count for `browser_sessions` (session + /// starts) and `product_events` (events). pub value: f64, } @@ -211,7 +212,8 @@ async fn flush_loop( } // Update pending gauge. Note: this now sums mixed units across - // features (GB for logs/traces/metrics, counts for browser_sessions); + // features (GB for logs/traces/metrics, counts for browser_sessions + // and product_events); // the metric name is kept as-is to avoid breaking existing dashboards. let total_pending: f64 = accumulator.values().map(PendingUsage::total).sum(); metrics::autumn_pending_gb(total_pending); diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 0a41ed5e8..73372453d 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -745,6 +745,55 @@ impl OrgRouting { /// never drift apart. const BROWSER_SESSIONS_FEATURE_ID: &str = "browser_sessions"; +/// The Autumn feature ID product events meter as — one unit per event, whether +/// it arrived on `/v1/events` or as a `type == "custom"` row on +/// `/v1/sessionEvents` (a browser `track()` call is the same product event as a +/// server-side one; only the transport differs). +const PRODUCT_EVENTS_FEATURE_ID: &str = "product_events"; + +/// Meter `value` units of `feature_id` around a WAL enqueue: reserve through +/// Autumn's atomic check+event lock, run `enqueue`, then confirm or release the +/// lock. When Autumn could not reserve (disabled or unavailable) the quantity is +/// recorded fail-open through the retrying tracker after the enqueue succeeds, +/// so provider outages never drop data or usage. `value <= 0` meters nothing. +/// +/// This is the one shape every count-metered handler uses (session starts on +/// the metadata endpoint, product events on both event endpoints); keeping it in +/// one place is what stops the reserve → enqueue → finalize ordering drifting +/// between them. +async fn metered_enqueue( + state: &AppState, + org_id: &str, + feature_id: &'static str, + value: f64, + enqueue: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + let reservation = reserve_autumn_usage(state, org_id, feature_id, value).await?; + let enqueue_result = enqueue().await; + + if let (Some(entitlements), Some(reservation)) = + (&state.autumn_entitlements, reservation.as_ref()) + { + let _ = entitlements + .finalize(reservation, enqueue_result.is_ok()) + .await; + } + let accepted = enqueue_result?; + + // Fail-open fallback: if Autumn could not reserve, record after the WAL + // commit through the retrying tracker. + if reservation.is_none() && org_id != SENTINEL_ORG_ID && value > 0.0 { + if let Some(tracker) = &state.autumn_tracker { + tracker.track(org_id, feature_id, value); + } + } + Ok(accepted) +} + /// The 402 Autumn's entitlement check produces for `feature_id`, or `None` when /// the org may ingest it. /// @@ -2385,49 +2434,28 @@ async fn handle_replay_meta_inner( return Ok(0); } let count = rows.len(); - let reservation = reserve_autumn_usage( + metered_enqueue( state, &org_id, BROWSER_SESSIONS_FEATURE_ID, session_starts as f64, + || async { + pipeline + .accept_rows_to( + &org_id, + state.config.tinybird.datasource_session_replays.clone(), + rows, + TelemetrySignal::SessionReplays, + destination, + ) + .await + .map_err(|e| { + warn!(org_id = %org_id, error = %e, "session metadata enqueue rejected"); + api_error_from_pipeline(&e) + }) + }, ) .await?; - let enqueue_result = pipeline - .accept_rows_to( - &org_id, - state.config.tinybird.datasource_session_replays.clone(), - rows, - TelemetrySignal::SessionReplays, - destination, - ) - .await - .map_err(|e| { - warn!(org_id = %org_id, error = %e, "session metadata enqueue rejected"); - api_error_from_pipeline(&e) - }); - - if let Err(error) = enqueue_result { - if let (Some(entitlements), Some(reservation)) = - (&state.autumn_entitlements, reservation.as_ref()) - { - let _ = entitlements.finalize(reservation, false).await; - } - return Err(error); - } - - if let (Some(entitlements), Some(reservation)) = - (&state.autumn_entitlements, reservation.as_ref()) - { - let _ = entitlements.finalize(reservation, true).await; - } - - // Fail-open fallback: if Autumn could not reserve, record after the WAL - // commit through the retrying tracker. - if reservation.is_none() && org_id != SENTINEL_ORG_ID && session_starts > 0 { - if let Some(tracker) = &state.autumn_tracker { - tracker.track(&org_id, "browser_sessions", session_starts as f64); - } - } Ok(count) } @@ -2456,6 +2484,7 @@ async fn handle_session_events( "maple.ingest.clickhouse_ready" = tracing::field::Empty, "maple.ingest.destination" = tracing::field::Empty, "maple.session_events.dropped" = tracing::field::Empty, + "maple.product_events.metered" = tracing::field::Empty, ); let span_handle = span.clone(); match handle_session_events_inner(&state, &headers, body) @@ -2500,13 +2529,16 @@ async fn handle_session_events_inner( let destination = native_destination_for(&resolved_key); Span::current().record("maple.ingest.destination", destination.as_str()); - // Same Autumn gate as the metadata endpoint. Session events - // are not separately metered — `browser_sessions` remains the billed unit, - // and introducing a `browser_events` meter is a pricing decision, not a - // schema one — but they must still be entitlement-gated: an out-of-quota org - // whose metadata rows are rejected while its event stream keeps writing is - // the incoherent half of the old design, and it only widens now that custom - // events are a promoted feature. + // Same Autumn gate as the metadata endpoint. Automatic session events + // (clicks, navigations, errors, ...) are not separately metered — + // `browser_sessions` remains their billed unit — but they must still be + // entitlement-gated: an out-of-quota org whose metadata rows are rejected + // while its event stream keeps writing is the incoherent half of the old + // design. `type == "custom"` rows are different: a browser `track()` call is + // a product event, and it is metered as `product_events` below (same unit as + // `/v1/events`). The REJECTION here deliberately stays on `browser_sessions`: + // an exhausted product-events allowance is billed as usage_based overage and + // must not 402 a whole session transcript. if org_id != SENTINEL_ORG_ID { if let Some(error) = entitlement_rejection(state, &org_id, BROWSER_SESSIONS_FEATURE_ID).await @@ -2525,6 +2557,7 @@ async fn handle_session_events_inner( // metadata, org_id is taken from the authenticated key, never the body. let mut rows: Vec> = Vec::new(); let mut dropped: u64 = 0; + let mut custom_events: u64 = 0; for line in body.split(|&b| b == b'\n') { if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -2542,6 +2575,9 @@ async fn handle_session_events_inner( dropped += 1; continue; } + if obj.get("type").and_then(|v| v.as_str()) == Some("custom") { + custom_events += 1; + } obj.insert( "org_id".to_string(), serde_json::Value::String(org_id.clone()), @@ -2565,19 +2601,32 @@ async fn handle_session_events_inner( return Ok(0); } let count = rows.len(); - pipeline - .accept_rows_to( - &org_id, - state.config.tinybird.datasource_session_events.clone(), - rows, - TelemetrySignal::SessionEvents, - destination, - ) - .await - .map_err(|e| { - warn!(org_id = %org_id, error = %e, "session events enqueue rejected"); - api_error_from_pipeline(&e) - })?; + Span::current().record("maple.product_events.metered", custom_events); + // Only the custom rows are metered; the automatic ones ride on the + // session's `browser_sessions` unit. A batch with no custom rows reserves + // nothing (`metered_enqueue` skips zero) and just enqueues. + metered_enqueue( + state, + &org_id, + PRODUCT_EVENTS_FEATURE_ID, + custom_events as f64, + || async { + pipeline + .accept_rows_to( + &org_id, + state.config.tinybird.datasource_session_events.clone(), + rows, + TelemetrySignal::SessionEvents, + destination, + ) + .await + .map_err(|e| { + warn!(org_id = %org_id, error = %e, "session events enqueue rejected"); + api_error_from_pipeline(&e) + }) + }, + ) + .await?; Ok(count) } @@ -2605,6 +2654,7 @@ async fn handle_product_events( "maple.ingest.clickhouse_ready" = tracing::field::Empty, "maple.ingest.destination" = tracing::field::Empty, "maple.product_events.dropped" = tracing::field::Empty, + "maple.product_events.metered" = tracing::field::Empty, ); let span_handle = span.clone(); match handle_product_events_inner(&state, &headers, body) @@ -2633,9 +2683,10 @@ async fn handle_product_events( /// `POST /v1/events` — product events posted directly by backends and mobile /// apps (browser rows reach `product_events` through the `session_events` -/// materialized view instead). Same auth, entitlement gate, NDJSON framing and -/// per-row drop policy as `/v1/sessionEvents`; the row shape is fixed by -/// `sanitize_product_event`. +/// materialized view instead). Same auth, NDJSON framing and per-row drop +/// policy as `/v1/sessionEvents`; the row shape is fixed by +/// `sanitize_product_event`. Entitlement-gated and metered as `product_events`, +/// one unit per row that reaches the WAL. async fn handle_product_events_inner( state: &AppState, headers: &HeaderMap, @@ -2654,12 +2705,11 @@ async fn handle_product_events_inner( let destination = native_destination_for(&resolved_key); Span::current().record("maple.ingest.destination", destination.as_str()); - // Product events reuse the browser-sessions entitlement (the plan doc's - // default) and, like session events, are not separately metered — a - // `product_events` meter is a pricing decision, not a schema one. + // Product events are their own metered feature. The gate here catches + // "no active subscription" / hard-capped orgs before the body is parsed; + // the exact quantity is reserved per accepted row below. if org_id != SENTINEL_ORG_ID { - if let Some(error) = - entitlement_rejection(state, &org_id, BROWSER_SESSIONS_FEATURE_ID).await + if let Some(error) = entitlement_rejection(state, &org_id, PRODUCT_EVENTS_FEATURE_ID).await { return Err(error); } @@ -2713,20 +2763,32 @@ async fn handle_product_events_inner( if rows.is_empty() { return Ok(0); } + // Metered quantity = rows actually enqueued, i.e. after the sanitiser's + // drops — a malformed line is neither stored nor billed. let count = rows.len(); - pipeline - .accept_rows_to( - &org_id, - state.config.tinybird.datasource_product_events.clone(), - rows, - TelemetrySignal::ProductEvents, - destination, - ) - .await - .map_err(|e| { - warn!(org_id = %org_id, error = %e, "product events enqueue rejected"); - api_error_from_pipeline(&e) - })?; + Span::current().record("maple.product_events.metered", count as u64); + metered_enqueue( + state, + &org_id, + PRODUCT_EVENTS_FEATURE_ID, + count as f64, + || async { + pipeline + .accept_rows_to( + &org_id, + state.config.tinybird.datasource_product_events.clone(), + rows, + TelemetrySignal::ProductEvents, + destination, + ) + .await + .map_err(|e| { + warn!(org_id = %org_id, error = %e, "product events enqueue rejected"); + api_error_from_pipeline(&e) + }) + }, + ) + .await?; Ok(count) } @@ -6721,6 +6783,237 @@ mod tests { let _ = std::fs::remove_dir_all(&queue_dir); } + /// One request a fake Autumn saw: which endpoint, and the JSON body. + #[derive(Debug)] + struct AutumnCall { + path: String, + body: serde_json::Value, + } + + impl AutumnCall { + fn feature_id(&self) -> &str { + self.body["feature_id"].as_str().unwrap_or_default() + } + /// `required_balance` is only sent by `reserve`; a plain `is_allowed` + /// gate omits it. That is how the tests tell the two apart. + fn reserved_value(&self) -> Option { + self.body.get("required_balance").and_then(|v| v.as_f64()) + } + } + + async fn fake_autumn( + axum::extract::State(tx): axum::extract::State< + tokio::sync::mpsc::UnboundedSender, + >, + Path(path): Path, + body: Bytes, + ) -> axum::Json { + let body: serde_json::Value = serde_json::from_slice(&body).unwrap_or_default(); + let _ = tx.send(AutumnCall { path, body }); + axum::Json(serde_json::json!({ "allowed": true })) + } + + /// Spawn a fake Autumn that allows everything and records every call, and + /// point `state` at it with billing enforcement enabled. + async fn with_fake_autumn( + mut state: AppState, + ) -> (AppState, tokio::sync::mpsc::UnboundedReceiver) { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let app = Router::new() + .route("/v1/{*path}", post(fake_autumn)) + .with_state(tx); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + state.autumn_entitlements = Some(AutumnEntitlements::new( + state.http_client.clone(), + "am_sk_test".to_string(), + &format!("http://{addr}"), + )); + (state, rx) + } + + /// Drain everything the fake Autumn has seen so far. Reserve → enqueue → + /// finalize all complete before the handler returns, so no waiting is + /// needed once the handler future has resolved. + fn drain_autumn_calls( + rx: &mut tokio::sync::mpsc::UnboundedReceiver, + ) -> Vec { + let mut calls = Vec::new(); + while let Ok(call) = rx.try_recv() { + calls.push(call); + } + calls + } + + fn bearer_headers(raw_key: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + "authorization", + format!("Bearer {raw_key}").parse().unwrap(), + ); + headers + } + + #[tokio::test] + async fn product_events_endpoint_meters_each_enqueued_row_as_product_events() { + let queue_dir = unique_main_test_dir("product-events-meter"); + let state = + replay_blob_test_state("maple_sk_test_pe_meter", "org_pe_meter", queue_dir.clone()) + .await; + let (state, mut rx) = with_fake_autumn(state).await; + + // Three valid rows and one the sanitiser drops (reserved `$`-prefixed + // name). The dropped row is neither stored nor billed. + let body = concat!( + r#"{"name":"plan_started"}"#, + "\n", + r#"{"name":"$not_allowed"}"#, + "\n", + r#"{"name":"checkout_viewed","source":"mobile"}"#, + "\n", + r#"{"name":"$screen","source":"mobile","page_path":"Home"}"#, + "\n", + ); + let accepted = handle_product_events_inner( + &state, + &bearer_headers("maple_sk_test_pe_meter"), + Bytes::from_static(body.as_bytes()), + ) + .await + .expect("product events should be accepted"); + assert_eq!(accepted, 3); + + let calls = drain_autumn_calls(&mut rx); + let checks: Vec<&AutumnCall> = calls + .iter() + .filter(|c| c.path == "balances.check") + .collect(); + // Every check on this endpoint — the gate and the reservation — is + // against `product_events`, never `browser_sessions`. + assert!(!checks.is_empty(), "expected Autumn checks, saw {calls:?}"); + for check in &checks { + assert_eq!(check.feature_id(), "product_events", "{check:?}"); + } + let reservations: Vec = checks.iter().filter_map(|c| c.reserved_value()).collect(); + assert_eq!( + reservations, + vec![3.0], + "exactly one reservation for the enqueued row count" + ); + let finalizes: Vec<&AutumnCall> = calls + .iter() + .filter(|c| c.path == "balances.finalize") + .collect(); + assert_eq!(finalizes.len(), 1, "{calls:?}"); + assert_eq!(finalizes[0].body["action"], "confirm"); + + let _ = std::fs::remove_dir_all(&queue_dir); + } + + #[tokio::test] + async fn custom_session_events_are_metered_as_product_events_but_gated_on_browser_sessions() { + let queue_dir = unique_main_test_dir("session-events-custom-meter"); + let state = + replay_blob_test_state("maple_sk_test_se_meter", "org_se_meter", queue_dir.clone()) + .await; + let (state, mut rx) = with_fake_autumn(state).await; + + // Two `track()` calls, one automatic click, one unknown type (dropped). + let body = concat!( + r#"{"type":"custom","message":"signup_completed"}"#, + "\n", + r#"{"type":"click","message":"button#buy"}"#, + "\n", + r#"{"type":"custom","message":"plan_selected","attributes":{"plan":"pro"}}"#, + "\n", + r#"{"type":"not-a-real-type"}"#, + "\n", + ); + let accepted = handle_session_events_inner( + &state, + &bearer_headers("maple_sk_test_se_meter"), + Bytes::from_static(body.as_bytes()), + ) + .await + .expect("session events should be accepted"); + assert_eq!( + accepted, 3, + "custom + click rows are stored, unknown is dropped" + ); + + let calls = drain_autumn_calls(&mut rx); + let checks: Vec<&AutumnCall> = calls + .iter() + .filter(|c| c.path == "balances.check") + .collect(); + + // The entitlement gate (no `required_balance`) stays on browser_sessions: + // an exhausted product-events allowance must not 402 a whole transcript. + let gates: Vec<&str> = checks + .iter() + .filter(|c| c.reserved_value().is_none()) + .map(|c| c.feature_id()) + .collect(); + assert_eq!(gates, vec!["browser_sessions"], "{calls:?}"); + + // The reservation is for the two custom rows only, as product_events. + let reservations: Vec<(&str, f64)> = checks + .iter() + .filter_map(|c| c.reserved_value().map(|v| (c.feature_id(), v))) + .collect(); + assert_eq!(reservations, vec![("product_events", 2.0)], "{calls:?}"); + + let finalizes: Vec<&AutumnCall> = calls + .iter() + .filter(|c| c.path == "balances.finalize") + .collect(); + assert_eq!(finalizes.len(), 1, "{calls:?}"); + assert_eq!(finalizes[0].body["action"], "confirm"); + + let _ = std::fs::remove_dir_all(&queue_dir); + } + + #[tokio::test] + async fn session_events_without_custom_rows_reserve_nothing() { + let queue_dir = unique_main_test_dir("session-events-no-custom"); + let state = + replay_blob_test_state("maple_sk_test_se_auto", "org_se_auto", queue_dir.clone()).await; + let (state, mut rx) = with_fake_autumn(state).await; + + let body = concat!( + r#"{"type":"click","message":"a"}"#, + "\n", + r#"{"type":"navigation","message":"/pricing"}"#, + "\n", + ); + let accepted = handle_session_events_inner( + &state, + &bearer_headers("maple_sk_test_se_auto"), + Bytes::from_static(body.as_bytes()), + ) + .await + .expect("session events should be accepted"); + assert_eq!(accepted, 2); + + let calls = drain_autumn_calls(&mut rx); + // Automatic events ride on the session's browser_sessions unit: only + // the gate fires, no reservation and no finalize. + assert!( + calls + .iter() + .all(|c| c.path == "balances.check" && c.reserved_value().is_none()), + "{calls:?}" + ); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].feature_id(), "browser_sessions"); + + let _ = std::fs::remove_dir_all(&queue_dir); + } #[tokio::test] async fn replay_chunks_stay_inline_when_no_blob_store_is_configured() { // The self-hosted / BYO-ClickHouse path, and the pre-cutover managed diff --git a/apps/landing/messages/en.json b/apps/landing/messages/en.json index 59889e1e4..d477e1dae 100644 --- a/apps/landing/messages/en.json +++ b/apps/landing/messages/en.json @@ -351,6 +351,7 @@ "pricing_logs": "Logs", "pricing_traces": "Traces", "pricing_metrics": "Metrics", + "pricing_product_events": "Product Events", "pricing_start_trial": "Start {duration}-day free trial", "pricing_everything_included": "Everything is included.", "pricing_zero_host": "per host", @@ -359,6 +360,7 @@ "pricing_included_monthly": "Included every month", "pricing_rate_gb": "then {rate} / GB", "pricing_rate_session": "then {rate} / session", + "pricing_rate_events": "then {rate} / {block} events", "pricing_trial_reassure": "Free for {duration} days · Cancel anytime · Card required to start", "pricing_enterprise_rail": "Higher volume, custom retention, priority support.", "pricing_estimate_link": "Estimate your bill →", diff --git a/apps/landing/messages/ja.json b/apps/landing/messages/ja.json index 689d37354..fbb38a936 100644 --- a/apps/landing/messages/ja.json +++ b/apps/landing/messages/ja.json @@ -351,6 +351,7 @@ "pricing_logs": "\u30ed\u30b0", "pricing_traces": "\u30c8\u30ec\u30fc\u30b9", "pricing_metrics": "\u30e1\u30c8\u30ea\u30af\u30b9", + "pricing_product_events": "\u30d7\u30ed\u30c0\u30af\u30c8\u30a4\u30d9\u30f3\u30c8", "pricing_start_trial": "{duration}\u65e5\u9593\u306e\u7121\u6599\u30c8\u30e9\u30a4\u30a2\u30eb\u3092\u958b\u59cb", "pricing_everything_included": "\u3059\u3079\u3066\u306e\u6a5f\u80fd\u304c\u542b\u307e\u308c\u307e\u3059\u3002", "pricing_zero_host": "\u30db\u30b9\u30c8\u5358\u4f4d", @@ -359,6 +360,7 @@ "pricing_included_monthly": "\u6bce\u6708\u542b\u307e\u308c\u308b\u30c7\u30fc\u30bf", "pricing_rate_gb": "\u8d85\u904e\u5206 {rate} / GB", "pricing_rate_session": "\u8d85\u904e\u5206 {rate} / \u30bb\u30c3\u30b7\u30e7\u30f3", + "pricing_rate_events": "\u8d85\u904e\u5206 {rate} / {block} \u30a4\u30d9\u30f3\u30c8", "pricing_trial_reassure": "{duration}\u65e5\u9593\u7121\u6599 \u00b7 \u3044\u3064\u3067\u3082\u30ad\u30e3\u30f3\u30bb\u30eb\u53ef \u00b7 \u958b\u59cb\u306b\u306f\u30ab\u30fc\u30c9\u304c\u5fc5\u8981", "pricing_enterprise_rail": "\u5927\u5bb9\u91cf\u3001\u30ab\u30b9\u30bf\u30e0\u4fdd\u6301\u671f\u9593\u3001\u512a\u5148\u30b5\u30dd\u30fc\u30c8\u3002", "pricing_estimate_link": "\u8acb\u6c42\u984d\u3092\u8a66\u7b97 \u2192", diff --git a/apps/landing/messages/ko.json b/apps/landing/messages/ko.json index d193a240c..89447267b 100644 --- a/apps/landing/messages/ko.json +++ b/apps/landing/messages/ko.json @@ -351,6 +351,7 @@ "pricing_logs": "\ub85c\uadf8", "pricing_traces": "\ud2b8\ub808\uc774\uc2a4", "pricing_metrics": "\uba54\ud2b8\ub9ad", + "pricing_product_events": "\ud504\ub85c\ub355\ud2b8 \uc774\ubca4\ud2b8", "pricing_start_trial": "{duration}\uc77c \ubb34\ub8cc \uccb4\ud5d8 \uc2dc\uc791", "pricing_everything_included": "\ubaa8\ub4e0 \uae30\ub2a5\uc774 \ud3ec\ud568\ub429\ub2c8\ub2e4.", "pricing_zero_host": "\ud638\uc2a4\ud2b8\ub2f9", @@ -359,6 +360,7 @@ "pricing_included_monthly": "\ub9e4\uc6d4 \ud3ec\ud568", "pricing_rate_gb": "\ucd08\uacfc\ubd84 {rate} / GB", "pricing_rate_session": "\ucd08\uacfc\ubd84 {rate} / \uc138\uc158", + "pricing_rate_events": "\ucd08\uacfc\ubd84 {rate} / {block} \uc774\ubca4\ud2b8", "pricing_trial_reassure": "{duration}\uc77c \ubb34\ub8cc \u00b7 \uc5b8\uc81c\ub4e0 \ucde8\uc18c \u00b7 \uc2dc\uc791 \uc2dc \uce74\ub4dc \ud544\uc694", "pricing_enterprise_rail": "\ub300\uc6a9\ub7c9, \ub9de\ucda4 \ubcf4\uc874 \uae30\uac04, \uc6b0\uc120 \uc9c0\uc6d0.", "pricing_estimate_link": "\uccad\uad6c\uc561 \ucd94\uc815 \u2192", diff --git a/apps/landing/src/components/PricingTable.astro b/apps/landing/src/components/PricingTable.astro index fb19a97e0..36f27eae5 100644 --- a/apps/landing/src/components/PricingTable.astro +++ b/apps/landing/src/components/PricingTable.astro @@ -23,6 +23,7 @@ import { getOffer, money, platformFeatures, + rateBlock, rateLabel, volume, type Allotment, @@ -43,13 +44,17 @@ const PLATFORM_FEATURES = platformFeatures() // trailing "/ unit" one character apart and the rows read as misaligned. The // template gives the amount a fixed `ch` slot instead. Splitting on a sentinel // keeps every locale's own word order intact — ja renders "超過分 {rate} / セッション". +// Events are quoted per block ("then $0.05 / 1,000 events"): the per-event rate +// is a fraction of a cent and would render as "$0.000". const RATE_SLOT = "\u0000" const rowRate = (a: Allotment) => { if (a.rate === undefined) return undefined const rendered = a.unit === "gb" ? m.pricing_rate_gb({ rate: RATE_SLOT }) - : m.pricing_rate_session({ rate: RATE_SLOT }) + : a.unit === "sessions" + ? m.pricing_rate_session({ rate: RATE_SLOT }) + : m.pricing_rate_events({ rate: RATE_SLOT, block: rateBlock(a) }) const [before, after = ""] = rendered.split(RATE_SLOT) return { before, amount: rateLabel(a.rate), after } } @@ -177,6 +182,9 @@ const containerClass = variant === "home" ? "shell" : "mx-auto max-w-5xl px-6" {a.featureId === "browser_sessions" && ( )} + {a.featureId === "product_events" && ( + + )} {a.label}
diff --git a/apps/landing/src/content/docs/session-replay/product-events-api.md b/apps/landing/src/content/docs/session-replay/product-events-api.md index f24a6eb0d..0d8621626 100644 --- a/apps/landing/src/content/docs/session-replay/product-events-api.md +++ b/apps/landing/src/content/docs/session-replay/product-events-api.md @@ -53,7 +53,7 @@ Over-long strings are truncated at the caps above; unknown fields are discarded. | `200` | `{"accepted": }` — rows durably queued. Malformed rows (bad `name`, `source`, `timestamp`) are dropped individually and not counted. | | `400` | A line is not valid JSON, or not a JSON object. The whole batch is rejected. | | `401` | Missing or invalid ingest key. | -| `402` | The organization is out of quota for browser sessions (product events share that entitlement). | +| `402` | The organization is out of quota for product events (`product_events` is metered per event, separately from browser sessions). | | `503` | Storage temporarily unavailable — retry with backoff. | Product events are not metered separately: they are covered by the browser-sessions entitlement. diff --git a/apps/landing/src/lib/pricing-offer.ts b/apps/landing/src/lib/pricing-offer.ts index a895f76ba..e1afc077c 100644 --- a/apps/landing/src/lib/pricing-offer.ts +++ b/apps/landing/src/lib/pricing-offer.ts @@ -18,16 +18,24 @@ import * as m from "../paraglide/messages.js" const AUTUMN_API_VERSION = "2.3.0" -/** browser_sessions is metered per session; every other signal is per GB. */ -type Unit = "gb" | "count" +/** + * browser_sessions is metered per session, product_events per event; every + * other signal is per GB. + */ +export type Unit = "gb" | "sessions" | "events" export interface Allotment { featureId: string label: string unit: Unit included: number - /** Price per unit once `included` is used up. */ + /** Price per `rateUnits` units once `included` is used up. */ rate?: number + /** + * Units the rate is quoted per — 1 for GB and sessions, 1,000 for events. + * Autumn's `billingUnits`; the rate is the price of one such block. + */ + rateUnits: number } export interface Offer { @@ -43,14 +51,15 @@ export interface Offer { const HIDDEN_FEATURE_IDS = new Set(["ai_input_tokens", "ai_output_tokens"]) /** Canonical row order — Autumn can return items in any order. */ -const DATA_FEATURE_ORDER = ["logs", "traces", "metrics", "browser_sessions"] +const DATA_FEATURE_ORDER = ["logs", "traces", "metrics", "browser_sessions", "product_events"] const dataFeatureRank = (id: string | undefined) => { const i = id ? DATA_FEATURE_ORDER.indexOf(id) : -1 return i === -1 ? DATA_FEATURE_ORDER.length : i } -const unitFor = (featureId: string): Unit => (featureId === "browser_sessions" ? "count" : "gb") +const unitFor = (featureId: string): Unit => + featureId === "browser_sessions" ? "sessions" : featureId === "product_events" ? "events" : "gb" /** * Capitalized, localized labels for the metered rows, keyed by Autumn featureId @@ -61,6 +70,7 @@ const dataFeatureLabels = (): Record => ({ traces: m.pricing_traces(), metrics: m.pricing_metrics(), browser_sessions: m.nav_browser_sessions(), + product_events: m.pricing_product_events(), }) /** @@ -81,8 +91,8 @@ export const platformFeatures = (): string[] => [ /** * The live offer, or the `autumn.config.ts` mirror if Autumn is unreachable at - * build time. The fallback carries browser sessions too — a short fallback - * would render a meter that disagrees with the live one. + * build time. The fallback carries browser sessions and product events too — a + * short fallback would render a meter that disagrees with the live one. */ export async function getOffer(): Promise { const labels = dataFeatureLabels() @@ -124,6 +134,7 @@ export async function getOffer(): Promise { unit: unitFor(featureId), included: Number(item.included ?? 0), rate: item.price?.amount ?? undefined, + rateUnits: Math.max(1, item.price?.billingUnits ?? 1), } }) .sort((a, b) => dataFeatureRank(a.featureId) - dataFeatureRank(b.featureId)), @@ -143,15 +154,38 @@ export async function getOffer(): Promise { hasTrial: true, trialDuration: 14, allotments: [ - { featureId: "logs", label: labels.logs!, unit: "gb", included: 100, rate: 0.3 }, - { featureId: "traces", label: labels.traces!, unit: "gb", included: 100, rate: 0.3 }, - { featureId: "metrics", label: labels.metrics!, unit: "gb", included: 100, rate: 0.3 }, + { featureId: "logs", label: labels.logs!, unit: "gb", included: 100, rate: 0.3, rateUnits: 1 }, + { + featureId: "traces", + label: labels.traces!, + unit: "gb", + included: 100, + rate: 0.3, + rateUnits: 1, + }, + { + featureId: "metrics", + label: labels.metrics!, + unit: "gb", + included: 100, + rate: 0.3, + rateUnits: 1, + }, { featureId: "browser_sessions", label: labels.browser_sessions!, - unit: "count", + unit: "sessions", included: 5000, rate: 0.002, + rateUnits: 1, + }, + { + featureId: "product_events", + label: labels.product_events!, + unit: "events", + included: 1_000_000, + rate: 0.05, + rateUnits: 1000, }, ], ctaLabel: m.pricing_start_trial({ duration: "14" }), @@ -172,4 +206,11 @@ export const money = (n: number) => { /** Sub-cent rates (per session) need three places; per-GB rates need two. */ export const rateLabel = (n: number) => `$${n < 0.01 ? n.toFixed(3) : n.toFixed(2)}` +/** + * The block a rate is quoted per, for the "then {rate} / …" sentence: 1 for + * GB and sessions, "1,000" for events. Localized unit words come from the + * message catalog; this only supplies the number. + */ +export const rateBlock = (a: Allotment) => a.rateUnits.toLocaleString("en-US") + export const volume = (a: Allotment, n: number) => (a.unit === "gb" ? `${n} GB` : n.toLocaleString("en-US")) diff --git a/apps/landing/src/pages/pricing.md.ts b/apps/landing/src/pages/pricing.md.ts index 1f68d6466..1043d6109 100644 --- a/apps/landing/src/pages/pricing.md.ts +++ b/apps/landing/src/pages/pricing.md.ts @@ -8,10 +8,12 @@ */ import type { APIRoute } from "astro" import { blocks, docHeader, markdown, table } from "../lib/page-markdown" -import { getOffer, money, platformFeatures, rateLabel } from "../lib/pricing-offer" +import { getOffer, money, platformFeatures, rateBlock, rateLabel, type Allotment } from "../lib/pricing-offer" import * as m from "../paraglide/messages.js" -const unitLabel = (unit: "gb" | "count") => (unit === "gb" ? "GB" : "session") +/** "GB", "session", or "1,000 events" — the block the rate is quoted per. */ +const unitLabel = (a: Allotment) => + a.unit === "gb" ? "GB" : a.unit === "sessions" ? "session" : `${rateBlock(a)} events` export const GET: APIRoute = async () => { const offer = await getOffer() @@ -21,7 +23,7 @@ export const GET: APIRoute = async () => { offer.allotments.map((a) => [ a.label, a.unit === "gb" ? `${a.included} GB` : a.included.toLocaleString("en-US"), - a.rate === undefined ? "—" : `${rateLabel(a.rate)} / ${unitLabel(a.unit)}`, + a.rate === undefined ? "—" : `${rateLabel(a.rate)} / ${unitLabel(a)}`, ]), ) diff --git a/apps/web/src/components/icons/cursor-pointer.tsx b/apps/web/src/components/icons/cursor-pointer.tsx new file mode 100644 index 000000000..5b7162d76 --- /dev/null +++ b/apps/web/src/components/icons/cursor-pointer.tsx @@ -0,0 +1,27 @@ +import type { IconProps } from "./icon" + +function CursorPointerIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} +export { CursorPointerIcon } diff --git a/apps/web/src/components/icons/index.ts b/apps/web/src/components/icons/index.ts index 938662b8d..fd033a472 100644 --- a/apps/web/src/components/icons/index.ts +++ b/apps/web/src/components/icons/index.ts @@ -70,6 +70,7 @@ export { CloudflareIcon, CloudflareMonoIcon } from "./cloudflare" export { ChromeIcon } from "./chrome" export { CornerDownLeftIcon } from "./corner-down-left" export { CursorIcon } from "./cursor" +export { CursorPointerIcon } from "./cursor-pointer" export { CreditCardIcon } from "./credit-card" export { CubeIcon } from "./cube" export { DiscordIcon } from "./discord" diff --git a/apps/web/src/components/settings/feature-usage-cards.tsx b/apps/web/src/components/settings/feature-usage-cards.tsx index f22a3fe43..53bdebdc5 100644 --- a/apps/web/src/components/settings/feature-usage-cards.tsx +++ b/apps/web/src/components/settings/feature-usage-cards.tsx @@ -2,14 +2,21 @@ import { Skeleton } from "@maple/ui/components/ui/skeleton" import { cn } from "@maple/ui/lib/utils" import { formatCurrency } from "@/lib/billing/currency" -import { featureUnit, FEATURE_COLORS, type FeatureSpend, type SpendModel } from "@/lib/billing/spend" +import { + featureUnit, + FEATURE_COLORS, + formatRateLabel, + SPEND_FEATURES, + type FeatureSpend, + type SpendModel, +} from "@/lib/billing/spend" import { formatCount, formatUsage } from "@/lib/billing/usage" /** * One card per billable signal: how much was ingested, how much of it was * included, what the excess costs, and whether a cap is holding it back. * - * These sit above the spend chart on purpose — they are the four things the + * These sit above the spend chart on purpose — they are the five things the * customer is billed for, and the chart is only their sum over time. */ @@ -18,8 +25,8 @@ const formatVolume = (featureId: string, value: number) => export function FeatureUsageCardsSkeleton() { return ( -
- {Array.from({ length: 4 }).map((_, i) => ( +
+ {Array.from({ length: SPEND_FEATURES.length }).map((_, i) => (
@@ -56,7 +63,7 @@ function FeatureCard({ return ( // Fixed-height slots for the title and the headline, so the meter and the - // footer form one horizontal lane across all four cards. Height, not + // footer form one horizontal lane across all five cards. Height, not // nowrap: "Browser Sessions" is allowed to wrap to two lines (truncating it // to "Browse…" is worse), the slot just reserves the room whether it wraps // or not — so a one-word card doesn't pull its meter up. @@ -131,13 +138,7 @@ function FeatureCard({ )} - {hardCapped - ? "hard cap" - : feature.ratePerUnit === null - ? "—" - : featureUnit(feature.featureId) === "GB" - ? `$${feature.ratePerUnit.toFixed(2)}/GB` - : `$${feature.ratePerUnit}/session`} + {hardCapped ? "hard cap" : (formatRateLabel(feature) ?? "—")}
@@ -152,7 +153,7 @@ export function FeatureUsageCards({ overageCaps: Readonly> }) { return ( -
+
{model.features.map((feature) => ( .map((item) => { const unit = featureUnit(item.featureId as string) const amount = Number(item.included).toLocaleString() - return unit === "GB" ? `${amount} GB ${item.featureId}` : `${amount} sessions` + return unit === "GB" ? `${amount} GB ${item.featureId}` : `${amount} ${unit}` }) .join(" · ") diff --git a/apps/web/src/components/settings/pricing-cards.tsx b/apps/web/src/components/settings/pricing-cards.tsx index ba7b6e017..58f0127f5 100644 --- a/apps/web/src/components/settings/pricing-cards.tsx +++ b/apps/web/src/components/settings/pricing-cards.tsx @@ -45,6 +45,7 @@ import { CodeIcon, ShieldIcon, PlayRotateClockwiseIcon, + CursorPointerIcon, } from "@/components/icons" import type { IconComponent } from "@/components/icons" @@ -53,6 +54,7 @@ const FEATURE_ICONS: Record = { traces: PulseIcon, metrics: ChartLineIcon, browser_sessions: PlayRotateClockwiseIcon, + product_events: CursorPointerIcon, } satisfies Record // Display labels for the metered data rows, keyed by Autumn featureId (Autumn @@ -63,6 +65,13 @@ const DATA_FEATURE_LABELS: Record = { traces: "Traces", metrics: "Metrics", browser_sessions: "Browser Sessions", + product_events: "Product Events", +} satisfies Record + +// Count-metered features and their plural unit — everything else is GB. +const COUNT_UNITS: Record = { + browser_sessions: "sessions", + product_events: "events", } satisfies Record // Per-feature icons for the platform-feature rows, keyed by the `icon` strings @@ -103,8 +112,7 @@ function getPlanPrice(plan: Plan): { function formatIncludedUsage(item: PlanItem): string { if (item.unlimited) return "Unlimited" if (item.included != null) { - // browser_sessions is metered by count, not bytes — everything else is GB. - const unit = item.featureId === "browser_sessions" ? "sessions" : "GB" + const unit = (item.featureId ? COUNT_UNITS[item.featureId] : undefined) ?? "GB" return `${Number(item.included).toLocaleString()} ${unit}` } return "" @@ -133,6 +141,7 @@ const ENTERPRISE_DATA_FEATURES = [ { featureId: "traces", label: "Traces", value: "Custom" }, { featureId: "metrics", label: "Metrics", value: "Custom" }, { featureId: "browser_sessions", label: "Browser Sessions", value: "Custom" }, + { featureId: "product_events", label: "Product Events", value: "Custom" }, ] function getScenario(plan: Plan): string { diff --git a/apps/web/src/components/settings/spend-chart.tsx b/apps/web/src/components/settings/spend-chart.tsx index 0c2fe0db3..609c4e8ea 100644 --- a/apps/web/src/components/settings/spend-chart.tsx +++ b/apps/web/src/components/settings/spend-chart.tsx @@ -44,6 +44,10 @@ const chartConfig: ChartConfig = { label: FEATURE_SHORT_LABELS.browser_sessions, color: FEATURE_COLORS.browser_sessions, }, + product_events: { + label: FEATURE_SHORT_LABELS.product_events, + color: FEATURE_COLORS.product_events, + }, } const BANDS = ["base", ...SPEND_FEATURES] as const @@ -77,6 +81,7 @@ export function SpendChart({ model, daily }: { model: SpendModel; daily: DailySp traces: latest?.traces ?? 0, metrics: latest?.metrics ?? 0, browser_sessions: latest?.browser_sessions ?? 0, + product_events: latest?.product_events ?? 0, } satisfies Record<(typeof BANDS)[number], number> }, [data]) const lastPoint = data[data.length - 1] @@ -100,7 +105,7 @@ export function SpendChart({ model, daily }: { model: SpendModel; daily: DailySp

{/* The legend carries each band's cycle-to-date dollars, not just its - color: a row of five dots tells you which color is which, but not + color: a row of six dots tells you which color is which, but not which band is worth reading. With the amounts it doubles as the breakdown, and the "$0.00" bands say plainly that they contribute nothing rather than hiding somewhere on the axis. */} diff --git a/apps/web/src/lib/billing/cost-estimate.test.ts b/apps/web/src/lib/billing/cost-estimate.test.ts index 026c2133c..8eb0e204f 100644 --- a/apps/web/src/lib/billing/cost-estimate.test.ts +++ b/apps/web/src/lib/billing/cost-estimate.test.ts @@ -31,7 +31,7 @@ function buildCustomer( } // Mirrors apps/api/autumn.config.ts: $39/mo base, 100 GB included per signal at -// $0.30/GB overage, 5000 sessions at $0.002/session. +// $0.30/GB overage, 5000 sessions at $0.002/session, 1M events at $0.05/1,000. const startupPlan = { id: "startup", name: "Startup", @@ -42,6 +42,7 @@ const startupPlan = { { featureId: "traces", included: 100, price: { amount: 0.3, billingUnits: 1 } }, { featureId: "metrics", included: 100, price: { amount: 0.3, billingUnits: 1 } }, { featureId: "browser_sessions", included: 5000, price: { amount: 0.002, billingUnits: 1 } }, + { featureId: "product_events", included: 1_000_000, price: { amount: 0.05, billingUnits: 1000 } }, ], } as Plan @@ -93,6 +94,21 @@ describe("estimateCycleCost", () => { expect(estimate!.partial).toBe(false) }) + it("bills product events per 1,000-event block and says so in the detail", () => { + const estimate = estimateCycleCost({ + customer: buildCustomer([buildSubscription()], { product_events: { granted: 1_000_000 } }), + plans: [startupPlan], + usage: usage({ product_events: 1_250_500 }), + }) + const events = estimate!.lines.find((l) => l.key === "overage:product_events") + // 250,500 over → ceil(250.5) = 251 blocks × $0.05 + expect(events).toMatchObject({ + label: "Product Events overage", + detail: "250,500 events over included × $0.05 / 1,000 events", + }) + expect(events!.amount).toBeCloseTo(12.55) + }) + it("rounds overage up per billingUnits block", () => { const plan = { ...startupPlan, diff --git a/apps/web/src/lib/billing/cost-estimate.ts b/apps/web/src/lib/billing/cost-estimate.ts index 7ae8850ef..6b9c533f6 100644 --- a/apps/web/src/lib/billing/cost-estimate.ts +++ b/apps/web/src/lib/billing/cost-estimate.ts @@ -6,13 +6,20 @@ import { formatCount, formatUsage } from "./usage" // Metered features surfaced on the billing page. AI token features stay hidden, // matching HIDDEN_FEATURE_IDS in pricing-cards.tsx. -const METERED_FEATURES = ["logs", "traces", "metrics", "browser_sessions"] as const +const METERED_FEATURES = ["logs", "traces", "metrics", "browser_sessions", "product_events"] as const const FEATURE_LABELS: Record = { logs: "Logs", traces: "Traces", metrics: "Metrics", browser_sessions: "Browser Sessions", + product_events: "Product Events", +} satisfies Record + +// Count-metered features and their singular unit; everything else is per GB. +const COUNT_UNITS: Record = { + browser_sessions: "session", + product_events: "event", } satisfies Record export interface CostLine { @@ -34,15 +41,17 @@ export interface CycleCostEstimate { partial: boolean } -const featureUnit = (featureId: string): string => (featureId === "browser_sessions" ? "session" : "GB") +const featureUnit = (featureId: string): string => COUNT_UNITS[featureId] ?? "GB" // Sub-cent rates (e.g. $0.003/session) would round to "$0.00" through the // 2-decimal currency formatter, so render those raw. const formatRate = (rate: number): string => rate > 0 && rate < 0.01 ? `$${rate}` : formatCurrency(rate, "usd") -const formatQuantity = (featureId: string, value: number): string => - featureId === "browser_sessions" ? `${formatCount(value)} sessions` : formatUsage(value) +const formatQuantity = (featureId: string, value: number): string => { + const unit = COUNT_UNITS[featureId] + return unit === undefined ? formatUsage(value) : `${formatCount(value)} ${unit}s` +} /** * Actual-to-date cost estimate for the current billing cycle, computed purely diff --git a/apps/web/src/lib/billing/spend.test.ts b/apps/web/src/lib/billing/spend.test.ts index 03e66e81f..6d6ceea2d 100644 --- a/apps/web/src/lib/billing/spend.test.ts +++ b/apps/web/src/lib/billing/spend.test.ts @@ -7,7 +7,7 @@ import type { CatalogPlan, DailySpendResponse, } from "@maple/domain/http" -import { buildCumulativeSeries, buildSpendModel } from "./spend" +import { buildCumulativeSeries, buildSpendModel, featureUnit, formatRateLabel } from "./spend" // Mock builders construct only the consumed subset of each domain schema, the // same approach cost-estimate.test.ts takes. @@ -24,6 +24,8 @@ const startupPlan = { { featureId: "traces", included: 100, price: { amount: 0.3 } }, { featureId: "metrics", included: 100, price: { amount: 0.3 } }, { featureId: "browser_sessions", included: 5_000, price: { amount: 0.002 } }, + // Quoted per 1,000 events: `amount` is the price of one billing block. + { featureId: "product_events", included: 1_000_000, price: { amount: 0.05, billingUnits: 1_000 } }, ], } as CatalogPlan @@ -49,6 +51,7 @@ const buildCustomer = ( traces: { granted: 100 }, metrics: { granted: 100 }, browser_sessions: { granted: 5_000 }, + product_events: { granted: 1_000_000 }, }, }) as BillingCustomer @@ -141,6 +144,43 @@ describe("buildSpendModel", () => { expect(result?.partial).toBe(true) }) + it("prices product events per event from a rate quoted per 1,000", () => { + const result = buildSpendModel({ + customer: buildCustomer(), + plans: [startupPlan], + usage: { product_events: { sum: 1_400_000 } } as BillingUsage["total"], + nowMs: NOW, + }) + if (result === null) throw new Error("expected a model") + + const events = result.features.find((feature) => feature.featureId === "product_events") + if (events === undefined) throw new Error("expected product_events") + // 400,000 events over × $0.05 / 1,000 = $20 — NOT 400,000 × $0.05. + expect(events.overageUnits).toBe(400_000) + expect(events.overageCents).toBe(2_000) + expect(events.ratePerUnit).toBeCloseTo(0.00005, 10) + expect(events.billingUnits).toBe(1_000) + expect(result.topDriver?.featureId).toBe("product_events") + }) + + it("labels rates the way the price list quotes them", () => { + const result = model() + if (result === null) throw new Error("expected a model") + const labels = Object.fromEntries( + result.features.map((feature) => [feature.featureId, formatRateLabel(feature)]), + ) + expect(labels).toEqual({ + logs: "$0.30/GB", + traces: "$0.30/GB", + metrics: "$0.30/GB", + browser_sessions: "$0.002/session", + product_events: "$0.05/1,000 events", + }) + expect(featureUnit("product_events")).toBe("events") + expect(featureUnit("browser_sessions")).toBe("sessions") + expect(featureUnit("logs")).toBe("GB") + }) + it("falls back to the calendar month with no active subscription", () => { const result = buildSpendModel({ customer: buildCustomer({ subscriptions: [] }), diff --git a/apps/web/src/lib/billing/spend.ts b/apps/web/src/lib/billing/spend.ts index a2ef3eeda..079a88387 100644 --- a/apps/web/src/lib/billing/spend.ts +++ b/apps/web/src/lib/billing/spend.ts @@ -20,7 +20,7 @@ import { */ /** Features Maple meters, in the order they stack in the chart and read across the cards. */ -export const SPEND_FEATURES = ["logs", "traces", "metrics", "browser_sessions"] as const +export const SPEND_FEATURES = ["logs", "traces", "metrics", "browser_sessions", "product_events"] as const export type SpendFeatureId = (typeof SPEND_FEATURES)[number] export const FEATURE_LABELS: Record = { @@ -28,6 +28,7 @@ export const FEATURE_LABELS: Record = { traces: "Traces", metrics: "Metrics", browser_sessions: "Browser Sessions", + product_events: "Product Events", } satisfies Record /** @@ -38,27 +39,70 @@ export const FEATURE_LABELS: Record = { export const FEATURE_SHORT_LABELS: Record = { ...FEATURE_LABELS, browser_sessions: "Sessions", + product_events: "Events", } satisfies Record /** * Series colors, validated for contrast on the card surface and for deutan CVD * separation. Amber is deliberately absent: it belongs to the limit line, the * projection, and the brand — a data band in amber would read as one of those. + * + * `product_events` takes the cyan/teal slot of the shared chart palette. That + * hue lives on a different token per theme (`--chart-5` is cyan in light, + * purple in dark; `--chart-4` is purple in light, teal in dark), so it is picked + * per scheme rather than by one token name — the point is a stable hue that sits + * between the blue of logs and the green of traces without touching either. */ export const FEATURE_COLORS: Record = { logs: "#3987e5", traces: "#199e70", metrics: "#9085e9", browser_sessions: "#d55181", + product_events: "light-dark(var(--chart-5), var(--chart-4))", } satisfies Record -/** `browser_sessions` is metered per session; every other feature per GB. */ -export const featureUnit = (featureId: string): "GB" | "sessions" => - featureId === "browser_sessions" ? "sessions" : "GB" +/** Singular unit for the count-metered features; every other feature is per GB. */ +const COUNT_UNITS = { + browser_sessions: "session", + product_events: "event", +} as const satisfies Partial> + +type CountFeatureId = keyof typeof COUNT_UNITS +const isCountFeature = (featureId: string): featureId is CountFeatureId => featureId in COUNT_UNITS +const countUnitOf = (featureId: string) => (isCountFeature(featureId) ? COUNT_UNITS[featureId] : undefined) + +/** `browser_sessions` is metered per session and `product_events` per event; every other feature per GB. */ +export const featureUnit = (featureId: string): "GB" | "sessions" | "events" => { + const unit = countUnitOf(featureId) + return unit === undefined ? "GB" : `${unit}s` +} + +/** + * The overage rate as the catalog quotes it: per GB for the byte signals, per + * unit for sessions, and per `billingUnits` for events ("$0.05/1,000 events") — + * the per-event rate is a fraction of a cent and would read as "$0" if rendered + * through the currency formatter. + */ +export const formatRateLabel = (feature: FeatureSpend): string | null => { + if (feature.ratePerUnit === null) return null + const unit = countUnitOf(feature.featureId) + if (unit === undefined) return `$${feature.ratePerUnit.toFixed(2)}/GB` + if (feature.billingUnits > 1) { + const perBlock = feature.ratePerUnit * feature.billingUnits + return `$${Number(perBlock.toPrecision(6))}/${feature.billingUnits.toLocaleString("en-US")} ${unit}s` + } + return `$${feature.ratePerUnit}/${unit}` +} export interface FeatureSpend extends FeatureUsagePricing { readonly featureId: SpendFeatureId readonly label: string + /** + * Units the catalog quotes the rate per (1 for GB and sessions, 1,000 for + * events). `ratePerUnit` is already normalized to ONE unit; this is kept only + * so the rate can be shown the way the price list states it. + */ + readonly billingUnits: number readonly overageUnits: number readonly overageCents: number } @@ -147,13 +191,20 @@ export function buildSpendModel({ addOns.reduce((sum, addOn) => sum + (addOn.price?.amount ?? 0), 0) const pricing: Record = {} + const billingUnitsByFeature: Record = {} for (const featureId of SPEND_FEATURES) { const balance = customer.balances?.[featureId] const item = basePlan?.items?.find((entry) => entry.featureId === featureId) + // The catalog quotes `amount` per `billingUnits` (e.g. $0.05 per 1,000 + // events); every consumer here prices ONE unit, so normalize once. + const rawBillingUnits = item?.price?.billingUnits ?? 1 + const billingUnits = rawBillingUnits > 0 ? rawBillingUnits : 1 + billingUnitsByFeature[featureId] = billingUnits + const amount = item?.price?.amount pricing[featureId] = { used: usage?.[featureId]?.sum ?? 0, included: balance?.granted ?? item?.included ?? null, - ratePerUnit: item?.price?.amount ?? null, + ratePerUnit: amount == null ? null : amount / billingUnits, unlimited: balance?.unlimited === true, // A hard-capped feature bills no overage — see `billsOverage`. overageAllowed: balance?.overageAllowed, @@ -170,6 +221,7 @@ export function buildSpendModel({ ...entry, featureId, label: FEATURE_LABELS[featureId], + billingUnits: billingUnitsByFeature[featureId] ?? 1, overageUnits: overUnits, overageCents: spend.overageByFeature[featureId] ?? 0, } @@ -228,6 +280,7 @@ export interface CumulativePoint { readonly traces: number | null readonly metrics: number | null readonly browser_sessions: number | null + readonly product_events: number | null /** Cumulative total so far, for the tooltip. `null` for future days. */ readonly total: number | null /** @@ -276,7 +329,9 @@ export function buildCumulativeSeries({ ? day.tracesGB : featureId === "metrics" ? day.metricsGB - : day.browserSessions + : featureId === "browser_sessions" + ? day.browserSessions + : (day.productEvents ?? 0) const byFeature = new Map< SpendFeatureId, @@ -301,6 +356,7 @@ export function buildCumulativeSeries({ traces: 0, metrics: 0, browser_sessions: 0, + product_events: 0, } satisfies Record const todayMs = Math.floor(model.cycle.nowMs / DAY_MS) * DAY_MS @@ -323,7 +379,13 @@ export function buildCumulativeSeries({ } } - const total = baseDollars + accrued.logs + accrued.traces + accrued.metrics + accrued.browser_sessions + const total = + baseDollars + + accrued.logs + + accrued.traces + + accrued.metrics + + accrued.browser_sessions + + accrued.product_events // Two anchors only. Recharts joins them into one straight dashed segment, // which is the honest shape for a linear projection — a curve through @@ -339,6 +401,7 @@ export function buildCumulativeSeries({ traces: future ? null : accrued.traces, metrics: future ? null : accrued.metrics, browser_sessions: future ? null : accrued.browser_sessions, + product_events: future ? null : accrued.product_events, total: future ? null : total, projected, future, diff --git a/apps/web/src/lib/services/atoms/billing-atoms.ts b/apps/web/src/lib/services/atoms/billing-atoms.ts index 07c87e77f..221c0ed4e 100644 --- a/apps/web/src/lib/services/atoms/billing-atoms.ts +++ b/apps/web/src/lib/services/atoms/billing-atoms.ts @@ -22,10 +22,13 @@ export const billingPlansAtom = retainedQuery("billingPublic", "listPlans", { reactivityKeys: [BILLING_PLANS_KEY], }) -// The billing page always meters the same four features over one billing cycle, +// The billing page always meters the same five features over one billing cycle, // so a single static atom (not a family) is enough. export const billingUsageAtom = retainedInternalQuery("billing", "getUsage", { - query: { featureId: ["logs", "traces", "metrics", "browser_sessions"], range: "1bc" }, + query: { + featureId: ["logs", "traces", "metrics", "browser_sessions", "product_events"], + range: "1bc", + }, reactivityKeys: [BILLING_USAGE_KEY], }) diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index b66c59a6f..3b810978f 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -22,6 +22,27 @@ as a real funnel in the product rather than a hand-written `run_sql`. to set unless the datasource name differs. 5. Publish `@maple-dev/effect-sdk` / `@maple-dev/browser` so customers' events start carrying identity; older builds keep writing (all new columns default). +6. **Billing**: `bun run --cwd apps/api atmn push` (no package script exists; `atmn` is an + `apps/api` dependency) so the `product_events` feature and its `startup` plan item exist in + Autumn before the gateway starts reserving against it. Until pushed, `balances.check` for an + unknown feature fails open and usage is still tracked, so nothing is rejected — just unbilled. + +### Billing + +Product events are their own metered Autumn feature, `product_events` (unit = one event), +separate from `browser_sessions`. The ingest gateway meters it on two paths, both with the same +reserve → WAL enqueue → confirm/release shape as session starts (`metered_enqueue` in +`apps/ingest/src/main.rs`): (1) `POST /v1/events` reserves the number of rows that survived +`sanitize_product_event`, and its entitlement gate is `product_events`; (2) `POST /v1/sessionEvents` +reserves the number of `type == "custom"` rows in the batch (a browser `track()` call is the same +unit as a server-side event) but keeps its entitlement REJECTION on `browser_sessions`, so an +exhausted product-events allowance bills usage-based overage instead of 402-ing a whole session +transcript. Automatic session events (clicks, navigations, ...) stay unmetered. `startup` includes +1,000,000 events/month, then **$0.05 per 1,000 events — a placeholder chosen 2026-08-17 (PostHog-style +order of magnitude), to be confirmed** before the config is pushed to production. Not yet done: +`DailySpendService` does not emit `DailyVolume.productEvents` (the domain field is optional for +that reason), so the spend chart's product-events series reads as zero until a warehouse query +lands. ## Where we are @@ -126,8 +147,8 @@ ingest key (org from the key, never the body). Body per line: Reject `name` starting with `$` from direct ingest (reserved for `$pageview`/`$screen`). - `Kind = 'custom'` unless `name = '$screen'` (mobile screen views → `Kind='screen'`). - New `TelemetrySignal::ProductEvents`, `INGEST_TINYBIRD_DATASOURCE_PRODUCT_EVENTS`, entry in - `clickhouse_insert_mappings.rs`, entitlement check reusing the browser-sessions feature id (or a - new `product_events` feature — billing decision, default: reuse). + `clickhouse_insert_mappings.rs`, entitlement check + per-row metering against the new + `product_events` feature (see "Billing" above). - Writes go where session events go today (Tinybird managed; BYO CH via the export lane). - Mobile: no SDK in scope. The endpoint _is_ the contract; a mobile app posts with a persistent install id as `visitor_id` and `identify`-equivalent `user_id`. `@maple-dev/effect-sdk` server side @@ -226,7 +247,8 @@ row for 30d — the rename is a rebuild, not a `RENAME TABLE`. ## Open decisions (defaults chosen; flag if you disagree) - Retention **180d** for `product_events` (could be 365; cost is negligible either way). -- Reuse the browser-sessions entitlement for `/v1/events` rather than a new billable feature. +- ~~Reuse the browser-sessions entitlement for `/v1/events` rather than a new billable feature.~~ + Superseded 2026-08-17: `product_events` is its own metered feature (see "Billing"). - Person key = `UserId` else `VisitorId`, stitched through `identity_links`; no probabilistic matching. - `signup_completed` truth = Clerk webhook, not the client. diff --git a/packages/domain/src/http/billing.ts b/packages/domain/src/http/billing.ts index e276572af..ca2f488d0 100644 --- a/packages/domain/src/http/billing.ts +++ b/packages/domain/src/http/billing.ts @@ -80,7 +80,13 @@ export class BillingSubscription extends Schema.Class("Bill export const BillingLimitType = Schema.Literals(["absolute", "usage_percentage"]) export type BillingLimitType = typeof BillingLimitType.Type -export const BillingFeatureId = Schema.Literals(["logs", "traces", "metrics", "browser_sessions"]) +export const BillingFeatureId = Schema.Literals([ + "logs", + "traces", + "metrics", + "browser_sessions", + "product_events", +]) export type BillingFeatureId = typeof BillingFeatureId.Type export const BillingAlertThresholdType = Schema.Literals([ @@ -186,7 +192,7 @@ export class BillingUsageFeature extends Schema.Class("Bill }) {} export class BillingUsage extends Schema.Class("BillingUsage")({ - // Keyed by Autumn featureId (logs/traces/metrics/browser_sessions). + // Keyed by Autumn featureId (logs/traces/metrics/browser_sessions/product_events). total: Schema.optionalKey(Schema.Record(Schema.String, BillingUsageFeature)), }) {} @@ -208,6 +214,8 @@ export class DailyVolume extends Schema.Class("DailyVolume")({ tracesGB: Schema.Number, metricsGB: Schema.Number, browserSessions: Schema.Number, + /** Product events (browser `track()` + server events) metered that day. Absent until the API emits it. */ + productEvents: Schema.optionalKey(Schema.Number), }) {} export class DailySpendResponse extends Schema.Class("DailySpendResponse")({ diff --git a/packages/query-engine-integrations/src/product/billing-usage.test.ts b/packages/query-engine-integrations/src/product/billing-usage.test.ts index 59c146374..81748e6e5 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.test.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" import { compileCH, type CompiledQuery } from "@maple-dev/clickhouse-builder" import { + dailyProductEventCountQuery, + dailyProductEventCountRowSchema, dailySessionCountQuery, dailySessionCountRowSchema, dailySignalVolumeQuery, @@ -99,3 +101,27 @@ describe("dailySessionCountQuery", () => { expect(row).toEqual({ day: "2026-07-01 00:00:00", sessions: 1284 }) }) }) + +describe("dailyProductEventCountQuery", () => { + it("counts billable product events per UTC day, excluding page views", () => { + const { sql } = compileCH(dailyProductEventCountQuery(), params) + + expect(sql).toContain("FROM product_events") + expect(sql).toContain("OrgId = 'org_123'") + expect(sql).toContain("toStartOfInterval(Timestamp, INTERVAL 86400 SECOND) AS day") + expect(sql).toContain("count() AS events") + expect(sql).toContain("Kind != 'navigation'") + expect(sql).toContain("Timestamp >= toDateTime('2026-07-01 00:00:00')") + expect(sql).toContain("GROUP BY day") + }) + + it("decodes a string event count", () => { + const compiled = compileCH(dailyProductEventCountQuery(), params, { + rowSchema: dailyProductEventCountRowSchema, + }) + + const [row] = decodeRows(compiled, [{ day: "2026-07-01 00:00:00", events: "40120" }]) + + expect(row).toEqual({ day: "2026-07-01 00:00:00", events: 40120 }) + }) +}) diff --git a/packages/query-engine-integrations/src/product/billing-usage.ts b/packages/query-engine-integrations/src/product/billing-usage.ts index e74ae8a19..43135d6a5 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.ts @@ -21,7 +21,7 @@ import { Schema } from "effect" import * as CH from "@maple-dev/clickhouse-builder/expr" import { from, param, type CompiledQueryRowSchema } from "@maple-dev/clickhouse-builder" -import { ServiceUsage, SessionReplays } from "@maple/query-engine/ch/tables" +import { ProductEvents, ServiceUsage, SessionReplays } from "@maple/query-engine/ch/tables" import { CHNumber } from "@maple/query-engine/ch/schema" import { hourFloor } from "@maple/query-engine/ch/query-helpers" @@ -107,3 +107,41 @@ export function dailySessionCountQuery() { .orderBy(["day", "asc"]) .format("JSON") } + +export interface DailyProductEventCountOutput { + readonly day: string + readonly events: number +} + +export const dailyProductEventCountRowSchema: CompiledQueryRowSchema = + Schema.Struct({ + day: Schema.String, + events: CHNumber, + }) + +/** + * Per-UTC-day billable product events for one org. + * + * Mirrors what the ingest gateway meters as `product_events`: every directly + * posted row (`POST /v1/events`, `Kind` custom/screen) plus browser `track()` + * calls (`Kind = 'custom'` via `product_events_mv`). Page views are part of the + * session, billed under `browser_sessions`, so `Kind = 'navigation'` is + * excluded. `product_events` is PARTITION BY toDate(Timestamp) with Timestamp + * second in the sorting key, so the window predicate is a primary-index range. + */ +export function dailyProductEventCountQuery() { + return from(ProductEvents) + .select(($) => ({ + day: CH.toStartOfInterval($.Timestamp, DAY_SECONDS), + events: CH.count(), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(CH.toDateTime(param.dateTime("startTime"))), + $.Timestamp.lte(CH.toDateTime(param.dateTime("endTime"))), + $.Kind.neq("navigation"), + ]) + .groupBy("day") + .orderBy(["day", "asc"]) + .format("JSON") +} diff --git a/packages/query-engine-integrations/src/product/index.ts b/packages/query-engine-integrations/src/product/index.ts index 92220cbf9..f0a73e54a 100644 --- a/packages/query-engine-integrations/src/product/index.ts +++ b/packages/query-engine-integrations/src/product/index.ts @@ -40,6 +40,8 @@ export { } from "./setup-audit" export { + dailyProductEventCountQuery, + dailyProductEventCountRowSchema, dailySessionCountQuery, dailySessionCountRowSchema, dailySignalVolumeQuery, From 3ee3d45f6acb8d5289289aa16525521c9bbc5e31 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 14:02:10 +0200 Subject: [PATCH 12/15] feat(billing): product_events free + unlimited during beta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autumn item is unlimited with no price; usage is still metered/tracked. Landing pricing renders 'Unlimited · free during beta' (en/ja/ko, HTML + /pricing.md); web billing already handles unlimited/no-rate items. --- apps/api/autumn.config.ts | 18 +++++++----------- apps/landing/messages/en.json | 2 ++ apps/landing/messages/ja.json | 2 ++ apps/landing/messages/ko.json | 2 ++ apps/landing/src/components/PricingTable.astro | 3 +++ apps/landing/src/lib/pricing-offer.ts | 16 ++++++++++++---- apps/landing/src/pages/pricing.md.ts | 6 +++--- docs/product-events-funnels.md | 13 +++++++------ 8 files changed, 38 insertions(+), 24 deletions(-) diff --git a/apps/api/autumn.config.ts b/apps/api/autumn.config.ts index 41f2ffb18..66ccaf45d 100644 --- a/apps/api/autumn.config.ts +++ b/apps/api/autumn.config.ts @@ -108,18 +108,14 @@ export const startup = plan({ }, }, { + // BETA (2026-08-17): free and unlimited while product events are in + // beta. Usage is still metered by the gateway and tracked in Autumn so + // we know real volumes before pricing it. To start charging, replace + // `unlimited` with e.g. `included: 1_000_000` and a + // `price: { amount: 0.05, billingUnits: 1000, billingMethod: + // "usage_based", interval: "month" }` ($0.05 per 1,000 events). featureId: "product_events", - included: 1_000_000, - // PLACEHOLDER price, chosen 2026-08-17: $0.05 per 1,000 events past the - // first 1M/month (PostHog-style order of magnitude). `amount` is the - // price per `billingUnits` in atmn, so 0.05 / 1000 = $0.00005 per event. - // To be confirmed before the plan is pushed to production. - price: { - amount: 0.05, - billingUnits: 1000, - billingMethod: "usage_based", - interval: "month", - }, + unlimited: true, }, ], freeTrial: { diff --git a/apps/landing/messages/en.json b/apps/landing/messages/en.json index d477e1dae..474c4132e 100644 --- a/apps/landing/messages/en.json +++ b/apps/landing/messages/en.json @@ -361,6 +361,8 @@ "pricing_rate_gb": "then {rate} / GB", "pricing_rate_session": "then {rate} / session", "pricing_rate_events": "then {rate} / {block} events", + "pricing_unlimited": "Unlimited", + "pricing_free_beta": "free during beta", "pricing_trial_reassure": "Free for {duration} days · Cancel anytime · Card required to start", "pricing_enterprise_rail": "Higher volume, custom retention, priority support.", "pricing_estimate_link": "Estimate your bill →", diff --git a/apps/landing/messages/ja.json b/apps/landing/messages/ja.json index fbb38a936..d7f390f5c 100644 --- a/apps/landing/messages/ja.json +++ b/apps/landing/messages/ja.json @@ -361,6 +361,8 @@ "pricing_rate_gb": "\u8d85\u904e\u5206 {rate} / GB", "pricing_rate_session": "\u8d85\u904e\u5206 {rate} / \u30bb\u30c3\u30b7\u30e7\u30f3", "pricing_rate_events": "\u8d85\u904e\u5206 {rate} / {block} \u30a4\u30d9\u30f3\u30c8", + "pricing_unlimited": "\u7121\u5236\u9650", + "pricing_free_beta": "\u30d9\u30fc\u30bf\u671f\u9593\u4e2d\u306f\u7121\u6599", "pricing_trial_reassure": "{duration}\u65e5\u9593\u7121\u6599 \u00b7 \u3044\u3064\u3067\u3082\u30ad\u30e3\u30f3\u30bb\u30eb\u53ef \u00b7 \u958b\u59cb\u306b\u306f\u30ab\u30fc\u30c9\u304c\u5fc5\u8981", "pricing_enterprise_rail": "\u5927\u5bb9\u91cf\u3001\u30ab\u30b9\u30bf\u30e0\u4fdd\u6301\u671f\u9593\u3001\u512a\u5148\u30b5\u30dd\u30fc\u30c8\u3002", "pricing_estimate_link": "\u8acb\u6c42\u984d\u3092\u8a66\u7b97 \u2192", diff --git a/apps/landing/messages/ko.json b/apps/landing/messages/ko.json index 89447267b..28f5e1f9e 100644 --- a/apps/landing/messages/ko.json +++ b/apps/landing/messages/ko.json @@ -361,6 +361,8 @@ "pricing_rate_gb": "\ucd08\uacfc\ubd84 {rate} / GB", "pricing_rate_session": "\ucd08\uacfc\ubd84 {rate} / \uc138\uc158", "pricing_rate_events": "\ucd08\uacfc\ubd84 {rate} / {block} \uc774\ubca4\ud2b8", + "pricing_unlimited": "\ubb34\uc81c\ud55c", + "pricing_free_beta": "\ubca0\ud0c0 \uae30\uac04 \ub3d9\uc548 \ubb34\ub8cc", "pricing_trial_reassure": "{duration}\uc77c \ubb34\ub8cc \u00b7 \uc5b8\uc81c\ub4e0 \ucde8\uc18c \u00b7 \uc2dc\uc791 \uc2dc \uce74\ub4dc \ud544\uc694", "pricing_enterprise_rail": "\ub300\uc6a9\ub7c9, \ub9de\ucda4 \ubcf4\uc874 \uae30\uac04, \uc6b0\uc120 \uc9c0\uc6d0.", "pricing_estimate_link": "\uccad\uad6c\uc561 \ucd94\uc815 \u2192", diff --git a/apps/landing/src/components/PricingTable.astro b/apps/landing/src/components/PricingTable.astro index 36f27eae5..0b10d81af 100644 --- a/apps/landing/src/components/PricingTable.astro +++ b/apps/landing/src/components/PricingTable.astro @@ -48,6 +48,9 @@ const PLATFORM_FEATURES = platformFeatures() // is a fraction of a cent and would render as "$0.000". const RATE_SLOT = "\u0000" const rowRate = (a: Allotment) => { + // Unlimited beta rows have no overage: the cell carries the beta note in + // "then" position; the (empty) amount slot trails it harmlessly. + if (a.unlimited) return { before: m.pricing_free_beta(), amount: "", after: "" } if (a.rate === undefined) return undefined const rendered = a.unit === "gb" diff --git a/apps/landing/src/lib/pricing-offer.ts b/apps/landing/src/lib/pricing-offer.ts index e1afc077c..ee51d6ce0 100644 --- a/apps/landing/src/lib/pricing-offer.ts +++ b/apps/landing/src/lib/pricing-offer.ts @@ -29,6 +29,11 @@ export interface Allotment { label: string unit: Unit included: number + /** + * Autumn `unlimited`: no cap and no overage — the row reads "Unlimited · free + * during beta" instead of a number and a rate. `included`/`rate` are ignored. + */ + unlimited?: boolean /** Price per `rateUnits` units once `included` is used up. */ rate?: number /** @@ -133,6 +138,7 @@ export async function getOffer(): Promise { label: labels[featureId] ?? item.feature?.name ?? featureId, unit: unitFor(featureId), included: Number(item.included ?? 0), + unlimited: item.unlimited === true, rate: item.price?.amount ?? undefined, rateUnits: Math.max(1, item.price?.billingUnits ?? 1), } @@ -183,9 +189,10 @@ export async function getOffer(): Promise { featureId: "product_events", label: labels.product_events!, unit: "events", - included: 1_000_000, - rate: 0.05, - rateUnits: 1000, + // Free and unlimited during beta — mirrors `apps/api/autumn.config.ts`. + included: 0, + unlimited: true, + rateUnits: 1, }, ], ctaLabel: m.pricing_start_trial({ duration: "14" }), @@ -213,4 +220,5 @@ export const rateLabel = (n: number) => `$${n < 0.01 ? n.toFixed(3) : n.toFixed( */ export const rateBlock = (a: Allotment) => a.rateUnits.toLocaleString("en-US") -export const volume = (a: Allotment, n: number) => (a.unit === "gb" ? `${n} GB` : n.toLocaleString("en-US")) +export const volume = (a: Allotment, n: number) => + a.unlimited ? m.pricing_unlimited() : a.unit === "gb" ? `${n} GB` : n.toLocaleString("en-US") diff --git a/apps/landing/src/pages/pricing.md.ts b/apps/landing/src/pages/pricing.md.ts index 1043d6109..0db4d19e1 100644 --- a/apps/landing/src/pages/pricing.md.ts +++ b/apps/landing/src/pages/pricing.md.ts @@ -8,7 +8,7 @@ */ import type { APIRoute } from "astro" import { blocks, docHeader, markdown, table } from "../lib/page-markdown" -import { getOffer, money, platformFeatures, rateBlock, rateLabel, type Allotment } from "../lib/pricing-offer" +import { getOffer, money, platformFeatures, rateBlock, rateLabel, volume, type Allotment } from "../lib/pricing-offer" import * as m from "../paraglide/messages.js" /** "GB", "session", or "1,000 events" — the block the rate is quoted per. */ @@ -22,8 +22,8 @@ export const GET: APIRoute = async () => { ["Signal", "Included every month", "Then"], offer.allotments.map((a) => [ a.label, - a.unit === "gb" ? `${a.included} GB` : a.included.toLocaleString("en-US"), - a.rate === undefined ? "—" : `${rateLabel(a.rate)} / ${unitLabel(a)}`, + volume(a, a.included), + a.unlimited ? m.pricing_free_beta() : a.rate === undefined ? "—" : `${rateLabel(a.rate)} / ${unitLabel(a)}`, ]), ) diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index 3b810978f..fff95fb7c 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -37,12 +37,13 @@ reserve → WAL enqueue → confirm/release shape as session starts (`metered_en reserves the number of `type == "custom"` rows in the batch (a browser `track()` call is the same unit as a server-side event) but keeps its entitlement REJECTION on `browser_sessions`, so an exhausted product-events allowance bills usage-based overage instead of 402-ing a whole session -transcript. Automatic session events (clicks, navigations, ...) stay unmetered. `startup` includes -1,000,000 events/month, then **$0.05 per 1,000 events — a placeholder chosen 2026-08-17 (PostHog-style -order of magnitude), to be confirmed** before the config is pushed to production. Not yet done: -`DailySpendService` does not emit `DailyVolume.productEvents` (the domain field is optional for -that reason), so the spend chart's product-events series reads as zero until a warehouse query -lands. +transcript. Automatic session events (clicks, navigations, ...) stay unmetered. **Beta pricing +(2026-08-17): the `startup` item is `unlimited: true` with no price** — usage is tracked in Autumn +(and shown on the billing page as "Unlimited · free during beta") so real volumes are known before +a price is set. To start charging, swap `unlimited` for an `included` allowance + `price` in +`apps/api/autumn.config.ts` (the commented example is $0.05 per 1,000 events past 1M/month) and +push. `DailySpendService` emits `DailyVolume.productEvents` from `product_events` +(`Kind != 'navigation'`, matching what the gateway meters). ## Where we are From ccac6677fdc0df7b58dc9ebbb7fe9cfd455d6927 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 17 Aug 2026 17:18:07 +0200 Subject: [PATCH 13/15] fix(product-events): review fixes for funnels and metering - ingest: `metered_enqueue` takes an `OnDenied` policy. `/v1/sessionEvents` meters `product_events` fail-open, so a denied reservation (a customer with no balance for the feature yet, or a future hard cap) no longer 402s a whole session batch and takes the clicks and navigations beside the `track()` rows down with it. Session starts and `/v1/events` keep rejecting: there the metered feature IS the payload. - api: validate a funnel breakdown through the BREAKDOWN builder, so an out-of-range `limit` is a 400 instead of a defect thrown inside `compile`. - query-engine: a funnel whose only step is a `session` step is answered by the session-entry branch alone. The events branch had no predicate to filter on and read every `product_events` row in range to project zeros. - web: debounce the funnel definition before it reaches the atoms (every keystroke was its own `windowFunnel`), and only emit `attribute:` once the key is non-empty instead of breaking down by `Attributes['']`. --- .../src/routes/internal/query-engine.http.ts | 8 ++- apps/api/src/routes/query-helpers.ts | 10 ++- apps/ingest/src/main.rs | 43 +++++++++++-- .../funnels/analytics-funnels-view.tsx | 63 +++++++++++++++---- .../src/ch/queries/product-events.ts | 21 +++++-- 5 files changed, 121 insertions(+), 24 deletions(-) diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 065b8206c..5476201a3 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -1757,7 +1757,13 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("productEventsFunnelBreakdown", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - yield* validateFunnelDefinition(productEventsFunnelOpts(payload)) + // The breakdown builder, not the plain one: `limit` is validated + // only there, and an unvalidated one throws inside `compile`. + yield* validateFunnelDefinition({ + ...productEventsFunnelOpts(payload), + breakdownBy: payload.breakdownBy, + limit: payload.limit, + }) const rows = yield* runQuery(Queries.productEventsFunnelBreakdown, tenant, payload) return new ProductEventsFunnelBreakdownResponse({ data: rows.map((row) => ({ diff --git a/apps/api/src/routes/query-helpers.ts b/apps/api/src/routes/query-helpers.ts index 796831187..870d06c67 100644 --- a/apps/api/src/routes/query-helpers.ts +++ b/apps/api/src/routes/query-helpers.ts @@ -172,13 +172,19 @@ const isProductEventsFunnelError = Schema.is(CH.ProductEventsFunnelError) * and the reason lands in the 400 envelope. Anything else thrown is a genuine * defect and stays one. Shared by the internal endpoint and the share API's * `product_events_funnel` route plan. + * + * Breakdown options are checked through the BREAKDOWN builder: `limit` is + * validated there and nowhere else, so validating the plain funnel for a + * breakdown request would let `InvalidLimit` through to `compile` — the exact + * defect this helper exists to prevent. */ export const validateFunnelDefinition = ( - opts: CH.ProductEventsFunnelOpts, + opts: CH.ProductEventsFunnelOpts | CH.ProductEventsFunnelBreakdownOpts, ): Effect.Effect => Effect.suspend(() => { try { - CH.productEventsFunnelQuery(opts) + if ("breakdownBy" in opts) CH.productEventsFunnelBreakdownQuery(opts) + else CH.productEventsFunnelQuery(opts) return Effect.void } catch (error) { if (isProductEventsFunnelError(error)) { diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 73372453d..6c028626b 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -751,11 +751,28 @@ const BROWSER_SESSIONS_FEATURE_ID: &str = "browser_sessions"; /// server-side one; only the transport differs). const PRODUCT_EVENTS_FEATURE_ID: &str = "product_events"; +/// What a DENIED Autumn reservation means for the batch in flight. +#[derive(Clone, Copy, PartialEq, Eq)] +enum OnDenied { + /// 402 the request: the metered feature IS the payload, so an exhausted + /// allowance is a reason not to accept it (session starts, `/v1/events`). + Reject, + /// Keep the batch and record the usage fail-open. For a feature that is only + /// PART of the payload — the `type == "custom"` rows of a session-events + /// batch — a rejection would also drop the clicks, navigations and errors + /// beside them, which is the incoherent outcome the `browser_sessions` gate + /// exists to avoid. Autumn also answers `allowed: false` for a customer that + /// simply has no balance for the feature yet (a plan item not pushed, or not + /// granted to a live subscription), so denial here must never break ingest. + MeterAnyway, +} + /// Meter `value` units of `feature_id` around a WAL enqueue: reserve through /// Autumn's atomic check+event lock, run `enqueue`, then confirm or release the -/// lock. When Autumn could not reserve (disabled or unavailable) the quantity is -/// recorded fail-open through the retrying tracker after the enqueue succeeds, -/// so provider outages never drop data or usage. `value <= 0` meters nothing. +/// lock. When Autumn could not reserve (disabled, unavailable, or denied under +/// `OnDenied::MeterAnyway`) the quantity is recorded fail-open through the +/// retrying tracker after the enqueue succeeds, so provider outages never drop +/// data or usage. `value <= 0` meters nothing. /// /// This is the one shape every count-metered handler uses (session starts on /// the metadata endpoint, product events on both event endpoints); keeping it in @@ -766,13 +783,28 @@ async fn metered_enqueue( org_id: &str, feature_id: &'static str, value: f64, + on_denied: OnDenied, enqueue: F, ) -> Result where F: FnOnce() -> Fut, Fut: std::future::Future>, { - let reservation = reserve_autumn_usage(state, org_id, feature_id, value).await?; + // `reserve_autumn_usage` fails only on a denial (every other outcome is an + // `Ok(None)` fail-open), so swallowing the error here is exactly "denied". + let reservation = match reserve_autumn_usage(state, org_id, feature_id, value).await { + Ok(reservation) => reservation, + Err(error) => { + if on_denied == OnDenied::Reject { + return Err(error); + } + warn!( + org_id, + feature_id, value, "Autumn denied the reservation; metering fail-open" + ); + None + } + }; let enqueue_result = enqueue().await; if let (Some(entitlements), Some(reservation)) = @@ -2439,6 +2471,7 @@ async fn handle_replay_meta_inner( &org_id, BROWSER_SESSIONS_FEATURE_ID, session_starts as f64, + OnDenied::Reject, || async { pipeline .accept_rows_to( @@ -2610,6 +2643,7 @@ async fn handle_session_events_inner( &org_id, PRODUCT_EVENTS_FEATURE_ID, custom_events as f64, + OnDenied::MeterAnyway, || async { pipeline .accept_rows_to( @@ -2772,6 +2806,7 @@ async fn handle_product_events_inner( &org_id, PRODUCT_EVENTS_FEATURE_ID, count as f64, + OnDenied::Reject, || async { pipeline .accept_rows_to( diff --git a/apps/web/src/components/funnels/analytics-funnels-view.tsx b/apps/web/src/components/funnels/analytics-funnels-view.tsx index 7beaefc40..c6be92a44 100644 --- a/apps/web/src/components/funnels/analytics-funnels-view.tsx +++ b/apps/web/src/components/funnels/analytics-funnels-view.tsx @@ -1,6 +1,7 @@ -import type { ReactNode } from "react" +import { useMemo, useState, type ReactNode } from "react" import { Result } from "@/lib/effect-atom" +import { useDebouncedValue } from "@maple/ui/hooks/use-debounced-value" import { Input } from "@maple/ui/components/ui/input" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" import { Skeleton } from "@maple/ui/components/ui/skeleton" @@ -28,6 +29,7 @@ import { type FunnelDefinition, type FunnelKeyBy, type FunnelSessionDimension, + type FunnelStep, } from "./definition" import { FunnelStepBuilder } from "./funnel-step-builder" import { FunnelBreakdownTable, FunnelResults } from "./funnel-results" @@ -44,6 +46,9 @@ const ATTRIBUTE_PREFIX = "attribute:" const NONE = "__none__" const ATTRIBUTE = "__attribute__" +/** How long the definition may keep changing before it reaches the warehouse. */ +const DEFINITION_DEBOUNCE_MS = 400 + const KEY_BY_NOUN = { person: "persons", visitor: "visitors", @@ -86,8 +91,22 @@ export function AnalyticsFunnelsView({ .orElse(() => []) // Only complete steps go to the warehouse — a step still being typed would - // otherwise fire a query per keystroke and 400 on the blank name. - const steps = completedSteps(definition.steps) + // otherwise 400 on the blank name — and they go there DEBOUNCED: every + // keystroke in an event name or page path is a new atom key, i.e. its own + // `windowFunnel` aggregation (plus a breakdown), and `staleTime` cannot + // coalesce keys that never repeat. Debouncing a serialized key rather than the + // array keeps the comparison by value, so an unrelated re-render does not + // re-arm the timer. + const stepsKey = JSON.stringify(completedSteps(definition.steps)) + const debouncedStepsKey = useDebouncedValue(stepsKey, DEFINITION_DEBOUNCE_MS) + // SAFETY: `debouncedStepsKey` is only ever a value `stepsKey` held, i.e. a + // `JSON.stringify` of the `completedSteps()` array above, so it parses back to + // exactly that shape. + const steps = useMemo( + () => JSON.parse(debouncedStepsKey) as ReadonlyArray, + [debouncedStepsKey], + ) + const breakdownBy = useDebouncedValue(definition.breakdownBy, DEFINITION_DEBOUNCE_MS) const labels = steps.map(stepLabel) const unitNoun = KEY_BY_NOUN[definition.keyBy] @@ -226,7 +245,7 @@ export function AnalyticsFunnelsView({ }} labels={labels} unitNoun={unitNoun} - breakdownBy={definition.breakdownBy} + breakdownBy={breakdownBy} /> )}
@@ -310,6 +329,12 @@ function LabelledSelect({ label, children }: { label: string; children: ReactNod * Breakdown: none, one of the session dimensions, or `attribute:` typed * by hand — the attribute keys on `track()` events are the customer's own * vocabulary and there is no cheap way to list them. + * + * "Attribute…" and the key it needs are two interactions, so the mode is held + * here and only a NON-EMPTY key becomes a `breakdownBy`. Emitting the bare + * `attribute:` prefix would break the funnel down by `Attributes['']`, which no + * event carries: every person lands in the `(none)` group, and the aggregation + * that produced it was pure waste. */ function BreakdownPicker({ value, @@ -318,23 +343,36 @@ function BreakdownPicker({ value: FunnelBreakdownBy | undefined onChange: (value: FunnelBreakdownBy | undefined) => void }) { - const isAttribute = value !== undefined && value.startsWith(ATTRIBUTE_PREFIX) - const selected = value === undefined ? NONE : isAttribute ? ATTRIBUTE : value - const attributeKey = isAttribute ? value.slice(ATTRIBUTE_PREFIX.length) : "" + const fromValue = value !== undefined && value.startsWith(ATTRIBUTE_PREFIX) + const [attributeMode, setAttributeMode] = useState(fromValue) + const [attributeKey, setAttributeKey] = useState(fromValue ? value.slice(ATTRIBUTE_PREFIX.length) : "") + // The local mode only survives while nothing else is selected, so a URL that + // changes under us (Back, a shared link) wins over a stale "Attribute…". + const isAttribute = fromValue || (attributeMode && value === undefined) + const selected = isAttribute ? ATTRIBUTE : value === undefined ? NONE : value const items = { [NONE]: "None", ...FUNNEL_SESSION_DIMENSION_LABEL, [ATTRIBUTE]: "Attribute…", } + const emitAttribute = (key: string) => { + const trimmed = key.trim() + onChange(trimmed.length > 0 ? `${ATTRIBUTE_PREFIX}${trimmed}` : undefined) + } return ( onChange(`${ATTRIBUTE_PREFIX}${event.target.value}`)} + onChange={(event) => { + setAttributeKey(event.target.value) + emitAttribute(event.target.value) + }} placeholder="attribute key, e.g. plan" aria-label="Breakdown attribute key" className="w-40 font-mono text-xs" diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index 3960dca0d..1b97daf53 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -493,16 +493,25 @@ function sessionEntryBranch(plan: FunnelPlan, step: Extract GROUP BY key` — one row per person with the * deepest step they reached in order within the window. + * + * A funnel whose ONLY step is a `session` step has no event predicate at all, so + * the events branch would have nothing to filter on and would read every + * `product_events` row in range to project `s1..sN` as zeros — rows that can + * never reach a level. That funnel is answered by the session-entry branch + * alone, which is the state the step builder is in the moment step 1 becomes a + * session step, so the branch is dropped rather than scanned. */ function levelsQuery(plan: FunnelPlan) { const { opts } = plan - const events = eventsBranch(plan) + const hasEventStep = opts.steps.some((step) => step.kind !== "session") const source = plan.sessionStep - ? fromUnion( - unionAll(sessionEntryBranch(plan, plan.sessionStep), events), - "funnel_events", - ) - : fromQuery(events, "funnel_events") + ? hasEventStep + ? fromUnion( + unionAll(sessionEntryBranch(plan, plan.sessionStep), eventsBranch(plan)), + "funnel_events", + ) + : fromQuery(sessionEntryBranch(plan, plan.sessionStep), "funnel_events") + : fromQuery(eventsBranch(plan), "funnel_events") return source .select(($) => { From eeceb36a5b0484865d47298ec053b487533819e2 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 21 Aug 2026 19:15:17 +0200 Subject: [PATCH 14/15] fix(product-events): harden funnels for launch - identity_links: AggregatingMergeTree with min(FirstSeen) so merges keep the pair's earliest sighting; reader aggregates per pair before ranking - ingest: no entitlement gate on /v1/events and fail-open metering, so an un-provisioned product_events feature never 402s real traffic - ingest: drop client timestamps outside -30d/+1d of receipt (partition minting / immortal TTL rows) - mcp: validate funnel widget definitions on write with the builder's rules - ProductEventsService.track: total timeout budget on top of per-attempt timeouts; inline plan_started emit forked off the attach request --- apps/api/src/mcp/lib/dashboard-schema-doc.ts | 6 +- .../lib/validate-widget-renderability.test.ts | 50 +++++++++++++- .../mcp/lib/validate-widget-renderability.ts | 28 ++++++++ apps/api/src/routes/internal/billing.http.ts | 26 ++++++-- .../product-events/ProductEventsService.ts | 10 +++ apps/cli/src/server/local-schema-history.ts | 8 +-- apps/cli/src/server/schema-identity.ts | 2 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../cli/src/server/schema/local-schema-v6.sql | 6 +- apps/cli/src/server/schema/local-schema.sql | 6 +- apps/cli/test/local-store-migrations.test.ts | 10 ++- apps/cli/test/native-local-store-migration.sh | 2 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- apps/ingest/src/main.rs | 42 ++++++++---- apps/ingest/src/session_analytics.rs | 65 +++++++++++++++++++ docs/product-events-funnels.md | 16 +++-- .../src/ch/functions/aggregate.ts | 12 +++- .../migrations/0016_product_events.ts | 11 +++- .../domain/src/generated/clickhouse-schema.ts | 4 +- .../generated/tinybird-project-manifest.ts | 4 +- packages/domain/src/tinybird/datasources.ts | 25 +++++-- .../src/__sql_baseline__/catalog.sql | 36 +++++++++- .../src/ch/queries/product-events.test.ts | 6 +- .../src/ch/queries/product-events.ts | 30 +++++++-- packages/query-engine/src/ch/tables.ts | 7 +- 25 files changed, 344 insertions(+), 72 deletions(-) diff --git a/apps/api/src/mcp/lib/dashboard-schema-doc.ts b/apps/api/src/mcp/lib/dashboard-schema-doc.ts index cb392e94b..c46308615 100644 --- a/apps/api/src/mcp/lib/dashboard-schema-doc.ts +++ b/apps/api/src/mcp/lib/dashboard-schema-doc.ts @@ -274,7 +274,11 @@ const dataSourcesSection = (): string => "- `keyBy` — `person` (default; user id, else the visitor's linked user, else the visitor),", " `visitor`, `user`, or `session`.", "- `windowSeconds` — the whole chain must complete within this many seconds of step 1", - " (default 86400).", + " (default 86400). Must be positive.", + "", + "The step count, the step-1-only session rule and a positive `windowSeconds` are enforced on", + "write: a definition that breaks one of them is rejected rather than saved, because the", + "query engine would reject it again on every render.", "- `breakdownBy` — stored for parity with the /analytics Funnels view; the widget renders", " the unsegmented funnel. Use `query_funnel` for a breakdown.", "", diff --git a/apps/api/src/mcp/lib/validate-widget-renderability.test.ts b/apps/api/src/mcp/lib/validate-widget-renderability.test.ts index 8b105c624..e3fe4d97e 100644 --- a/apps/api/src/mcp/lib/validate-widget-renderability.test.ts +++ b/apps/api/src/mcp/lib/validate-widget-renderability.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest" -import { makeQueryDataSource, makeRawSqlDataSource, makeStaticDataSource } from "@maple/widgets/dashboard" +import { + makeProductEventsFunnelDataSource, + makeQueryDataSource, + makeRawSqlDataSource, + makeStaticDataSource, +} from "@maple/widgets/dashboard" import { makeQueryDraft } from "@/dashboard-templates/helpers" import type { PanelType } from "@maple/domain/http" import { collectDocumentRenderWarnings, validateWidgetRenderability } from "./validate-widget-renderability" @@ -66,6 +71,49 @@ describe("fatal — combinations that always render wrong", () => { expect(issues.fatal).toEqual([]) expect(issues.warnings).toEqual([]) }) + + // A funnel definition the query builder rejects would persist and then 400 on + // every render, signed-in and shared alike, with nothing in the authoring + // tool able to repair it. These are the builder's own three rules. + describe("a product-event funnel definition the builder cannot compile", () => { + const funnelWidget = (funnel: Record) => + widget("funnel", makeProductEventsFunnelDataSource({ steps: [] as never }), { funnel }) + const step = (eventName: string) => ({ kind: "event", eventName }) + + it("more than ten steps", () => { + const steps = Array.from({ length: 11 }, (_, i) => step(`e${i}`)) + expect(validateWidgetRenderability(funnelWidget({ steps })).fatal.join(" ")).toContain( + "at most 10 steps", + ) + }) + + it("a session step past step 1", () => { + const steps = [ + step("signup_completed"), + { kind: "session", dimension: "utmSource", value: "twitter" }, + ] + expect(validateWidgetRenderability(funnelWidget({ steps })).fatal.join(" ")).toContain( + "only valid as step 1", + ) + }) + + it("a non-positive conversion window", () => { + const steps = [step("a"), step("b")] + expect( + validateWidgetRenderability(funnelWidget({ steps, windowSeconds: 0 })).fatal.join(" "), + ).toContain("windowSeconds") + }) + + it("a valid definition, and a funnel with no steps at all, are clean", () => { + const steps = [{ kind: "session", dimension: "utmSource", value: "x" }, step("signup")] + expect(validateWidgetRenderability(funnelWidget({ steps, windowSeconds: 3600 })).fatal).toEqual( + [], + ) + // No steps: the widget is the original group-by breakdown drawn as a + // funnel, and none of these rules apply to it. + expect(validateWidgetRenderability(funnelWidget({ showStepPercent: true })).fatal).toEqual([]) + }) + }) }) describe("warnings — heuristics that must not block a restore", () => { diff --git a/apps/api/src/mcp/lib/validate-widget-renderability.ts b/apps/api/src/mcp/lib/validate-widget-renderability.ts index 1b49d53f7..2dd441269 100644 --- a/apps/api/src/mcp/lib/validate-widget-renderability.ts +++ b/apps/api/src/mcp/lib/validate-widget-renderability.ts @@ -7,6 +7,7 @@ import { type PanelType, } from "@maple/domain/http" import { dataSourceQuerySet, dataSourceRawSql, dataSourceTransform } from "@maple/widgets/dashboard" +import { FUNNEL_MAX_STEPS } from "@maple/query-model" import { isGroupByRequested } from "./inspect-widget" type DashboardWidget = typeof DashboardWidgetSchema.Type @@ -120,6 +121,33 @@ export const validateWidgetRenderability = (input: ValidateWidgetRenderabilityIn ) } + // A product-event funnel definition the query builder cannot compile. These + // are the same three rules `validate()` in `@maple/query-engine`'s + // `product-events.ts` enforces, and they have to run HERE, on the write, or + // the widget persists and then 400s on every render — signed-in and shared + // alike — with no way to repair it from the tool that created it. The + // /analytics view and the web widget builder both block on them already. + const funnelSteps = widget.display.funnel?.steps + if (funnelSteps !== undefined && funnelSteps.length > 0) { + if (funnelSteps.length > FUNNEL_MAX_STEPS) { + fatal.push( + `A product-event funnel has at most ${FUNNEL_MAX_STEPS} steps, but \`display_json.funnel.steps\` has ${funnelSteps.length}.`, + ) + } + const lateSession = funnelSteps.findIndex((step, index) => index > 0 && step.kind === "session") + if (lateSession !== -1) { + fatal.push( + `A \`{ "kind": "session" }\` funnel step describes how the session was acquired, so it is only valid as step 1 — \`display_json.funnel.steps\` has one at step ${lateSession + 1}.`, + ) + } + const windowSeconds = widget.display.funnel?.windowSeconds + if (windowSeconds !== undefined && (!Number.isFinite(windowSeconds) || windowSeconds <= 0)) { + fatal.push( + `\`display_json.funnel.windowSeconds\` is the conversion window and must be a positive number of seconds (got ${JSON.stringify(windowSeconds)}). Omit it to use the default 86400 (24h).`, + ) + } + } + // --- warnings ---------------------------------------------------------- // The breakdown endpoint is meaningless ungrouped: one bucket per time slice diff --git a/apps/api/src/routes/internal/billing.http.ts b/apps/api/src/routes/internal/billing.http.ts index 4c66f942f..e002a5c57 100644 --- a/apps/api/src/routes/internal/billing.http.ts +++ b/apps/api/src/routes/internal/billing.http.ts @@ -21,6 +21,7 @@ import { readCustomerCached, } from "@/services/billing/autumn-client" import { AutumnClient, type AutumnResult } from "@/services/billing/autumn-http" +import { forkRequestScoped } from "@/platform/fork-request-scoped" import { emitPlanStartedFromAttach } from "@/services/billing/plan-events" import { ProductEventsService } from "@/services/product-events/ProductEventsService" import { requireAdmin } from "@/services/auth/auth" @@ -184,12 +185,25 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleInternalApi, "billing", const attached = yield* decodeUpstream(AttachResult, response) // Inline (no-redirect) plan start; the Autumn webhook covers the // Stripe-checkout path. Never fails the request. - yield* emitPlanStartedFromAttach(productEvents, { - orgId: tenant.orgId, - userId: tenant.userId, - planId: payload.planId, - result: attached, - }) + // + // FORKED, unlike the webhook receivers: this is the Subscribe + // click, the one request where a stall reads as a failed payment, + // and `track` is a bounded-but-not-free POST to the ingest + // gateway. `forkRequestScoped` means a gateway that answers + // normally still gets the event (the fiber finishes long before + // the response is written) while a stalled one is interrupted at + // the response instead of holding the user. Losing it there costs + // nothing durable — `plan_events.ts` is explicit that this emit is + // the low-latency COMPLEMENT and the `billing.updated` webhook is + // the authoritative `plan_started`. + yield* forkRequestScoped( + emitPlanStartedFromAttach(productEvents, { + orgId: tenant.orgId, + userId: tenant.userId, + planId: payload.planId, + result: attached, + }), + ) return attached }), ) diff --git a/apps/api/src/services/product-events/ProductEventsService.ts b/apps/api/src/services/product-events/ProductEventsService.ts index e8fca80b6..2c0d1e6a9 100644 --- a/apps/api/src/services/product-events/ProductEventsService.ts +++ b/apps/api/src/services/product-events/ProductEventsService.ts @@ -165,6 +165,16 @@ export const makeProductEvents = (options: { const line = toProductEventLine(event, now) yield* post(line).pipe( Effect.retry({ times: RETRIES, while: isRetryable }), + // TOTAL budget, on top of the per-attempt timeout inside `post`. + // A timeout carries no `status`, so `isRetryable` treats it as + // retryable and an unreachable gateway would otherwise cost a caller + // two full timeouts back to back. `track` runs inline for the webhook + // receivers, so the ceiling has to be the one an inline caller can + // actually afford, not twice it. + Effect.timeoutOrElse({ + duration: REQUEST_TIMEOUT, + orElse: () => new ProductEventsError({ message: `Product event gave up (${event.name})` }), + }), Effect.catchCause((cause) => Effect.logDebug("Product event dropped").pipe( Effect.annotateLogs({ event: event.name, cause: String(cause) }), diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 4660b9f83..43de5dad3 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -60,9 +60,9 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje }), Object.freeze({ version: 6, - fingerprint: "4fb7062f1e068837", - digest: "4fb7062f1e068837ff72848af8e862a47155b7543a1de8401b7b67c9ce176792", - manifestDigest: "c11f1f5aa250a7ce7eb42b0680197f4a69bdbfd0ada82dbfc95865010e78d7cb", - projectRevision: "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04", + fingerprint: "18015521cc411d12", + digest: "18015521cc411d12bf37d5e0c61bd6db2ec8a498a773ef3c9b28172d957683be", + manifestDigest: "d5520602a0f598c99a6b40ad81657eb387c91ad2e297a3b96c8e4f6d87b2b128", + projectRevision: "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab", }), ] as const) diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index f7cdde738..7fb8f0bc1 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -28,7 +28,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" + "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 7f9ae745b..1182d98a5 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04", + "projectRevision": "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v6.sql b/apps/cli/src/server/schema/local-schema-v6.sql index 5745a92ef..feff21937 100644 --- a/apps/cli/src/server/schema/local-schema-v6.sql +++ b/apps/cli/src/server/schema/local-schema-v6.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04 +-- projectRevision: 71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab -- localSchemaVersion: 6 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -137,9 +137,9 @@ CREATE TABLE IF NOT EXISTS identity_links ( OrgId LowCardinality(String), VisitorId String, UserId String, - FirstSeen DateTime64(9) + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) ) -ENGINE = ReplacingMergeTree +ENGINE = AggregatingMergeTree PARTITION BY tuple() ORDER BY (OrgId, VisitorId, UserId) TTL toDate(FirstSeen) + INTERVAL 365 DAY; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 5745a92ef..feff21937 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04 +-- projectRevision: 71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab -- localSchemaVersion: 6 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -137,9 +137,9 @@ CREATE TABLE IF NOT EXISTS identity_links ( OrgId LowCardinality(String), VisitorId String, UserId String, - FirstSeen DateTime64(9) + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) ) -ENGINE = ReplacingMergeTree +ENGINE = AggregatingMergeTree PARTITION BY tuple() ORDER BY (OrgId, VisitorId, UserId) TTL toDate(FirstSeen) + INTERVAL 365 DAY; diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 6fff493fa..2dd8ab30d 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -59,8 +59,8 @@ import { join } from "node:path" describe("current local schema identity", () => { it("matches the generated v6 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("4fb7062f1e068837") - expect(SCHEMA_DIGEST).toBe("4fb7062f1e068837ff72848af8e862a47155b7543a1de8401b7b67c9ce176792") + expect(SCHEMA_FINGERPRINT).toBe("18015521cc411d12") + expect(SCHEMA_DIGEST).toBe("18015521cc411d12bf37d5e0c61bd6db2ec8a498a773ef3c9b28172d957683be") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) @@ -161,7 +161,11 @@ describe("current local schema identity", () => { expect(productEventsView?.definition).toContain("FROM session_events") expect(productEventsView?.definition).toContain("'browser' AS Source") const identityLinks = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "identity_links") - expect(identityLinks?.engine).toBe("ReplacingMergeTree") + // Aggregating, not Replacing: the funnel ranks a visitor's linked users by + // `FirstSeen`, so the merge has to keep the pair's EARLIEST sighting. A + // Replacing merge with no version column keeps an arbitrary duplicate and + // the ranking flips as merges land. + expect(identityLinks?.engine).toBe("AggregatingMergeTree") expect(identityLinks?.orderBy).toBe("(OrgId, VisitorId, UserId)") const identityLinksView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "identity_links_mv", diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index a667b54fb..ba1603c04 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 6 and .schema == "4fb7062f1e068837"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 6 and .schema == "18015521cc411d12"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 78e001988..0f5024b35 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04"; +pub const PROJECT_REVISION: &str = "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 6c028626b..9fa5f3543 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -2739,16 +2739,16 @@ async fn handle_product_events_inner( let destination = native_destination_for(&resolved_key); Span::current().record("maple.ingest.destination", destination.as_str()); - // Product events are their own metered feature. The gate here catches - // "no active subscription" / hard-capped orgs before the body is parsed; - // the exact quantity is reserved per accepted row below. - if org_id != SENTINEL_ORG_ID { - if let Some(error) = entitlement_rejection(state, &org_id, PRODUCT_EVENTS_FEATURE_ID).await - { - return Err(error); - } - } - + // Product events are their own metered feature, but they are NOT gated on + // it — same reasoning as the `type == "custom"` rows on `/v1/sessionEvents`. + // Autumn answers `allowed: false` for a customer that simply has no balance + // for the feature yet (a plan item not pushed, or not granted to a live + // subscription), which is every org until the `atmn push` in the rollout + // checklist lands. A gate here would turn that window into a 402 on every + // backend and mobile event — including the API's own signup/plan emits — + // and `decide_allowed` reads a well-formed `allowed: false` as a real + // denial, so the fail-open in `is_allowed` never rescues it. The quantity is + // still reserved and recorded per accepted row below. let pipeline = native_rows_pipeline_for( state, destination, @@ -2806,7 +2806,11 @@ async fn handle_product_events_inner( &org_id, PRODUCT_EVENTS_FEATURE_ID, count as f64, - OnDenied::Reject, + // Fail-open, matching the entitlement decision above: a denial here is + // far more likely to mean "the feature is not provisioned yet" than + // "this org is over its allowance", and dropping a backend's buffered + // batch is not a recoverable outcome for the caller. + OnDenied::MeterAnyway, || async { pipeline .accept_rows_to( @@ -6928,12 +6932,24 @@ mod tests { .iter() .filter(|c| c.path == "balances.check") .collect(); - // Every check on this endpoint — the gate and the reservation — is - // against `product_events`, never `browser_sessions`. + // Every check on this endpoint is against `product_events`, never + // `browser_sessions`. assert!(!checks.is_empty(), "expected Autumn checks, saw {calls:?}"); for check in &checks { assert_eq!(check.feature_id(), "product_events", "{check:?}"); } + // ...and there is NO entitlement gate (a check with no + // `required_balance`), only the reservation. A gate would 402 every org + // whose Autumn customer has no `product_events` balance yet, which is + // every org until the plan item is pushed and granted — Autumn answers + // that with a real `allowed: false`, not an error, so the fail-open in + // `is_allowed` does not cover it. + let gates: Vec<&str> = checks + .iter() + .filter(|c| c.reserved_value().is_none()) + .map(|c| c.feature_id()) + .collect(); + assert!(gates.is_empty(), "expected no entitlement gate, saw {gates:?}"); let reservations: Vec = checks.iter().filter_map(|c| c.reserved_value()).collect(); assert_eq!( reservations, diff --git a/apps/ingest/src/session_analytics.rs b/apps/ingest/src/session_analytics.rs index dd1217fa8..0f7c3e069 100644 --- a/apps/ingest/src/session_analytics.rs +++ b/apps/ingest/src/session_analytics.rs @@ -280,6 +280,21 @@ const PRODUCT_EVENT_ID_FIELDS: [(&str, usize); 5] = [ /// which client produced it. const PRODUCT_EVENT_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.9f"; +/// How far a client-supplied `timestamp` may sit from the moment the gateway +/// received it before the row is dropped. +/// +/// The timestamp is otherwise the one unbounded field on the row, and it is the +/// one that decides physical layout: `product_events` is +/// `PARTITION BY toDate(Timestamp)`, so a backfill stamped across five years of +/// history mints ~2,000 single-row partitions for that org, and a far-FUTURE +/// stamp is worse still — `TTL toDate(Timestamp) + INTERVAL 365 DAY` never +/// fires, so the row is resident forever while sitting outside every funnel +/// window that could show it. The window is asymmetric because the two +/// directions mean different things: trailing a little is a buffered backend +/// flush or a clock behind, leading is always a bug. +const PRODUCT_EVENT_MAX_BACKDATE_DAYS: i64 = 30; +const PRODUCT_EVENT_MAX_FUTURE_DAYS: i64 = 1; + /// Parse a client-supplied product-event timestamp. /// /// Accepts RFC 3339 (`2026-08-17T10:15:30.123Z`, offsets allowed) or the @@ -297,6 +312,20 @@ fn parse_product_event_timestamp(value: &str) -> Option, + received_at: chrono::DateTime, +) -> bool { + let behind = chrono::TimeDelta::try_days(PRODUCT_EVENT_MAX_BACKDATE_DAYS); + let ahead = chrono::TimeDelta::try_days(PRODUCT_EVENT_MAX_FUTURE_DAYS); + match (behind, ahead) { + (Some(behind), Some(ahead)) => at >= received_at - behind && at <= received_at + ahead, + // Unreachable for these constants; keep the row rather than invent a bound. + _ => true, + } +} + fn format_product_event_timestamp(at: chrono::DateTime) -> String { at.format(PRODUCT_EVENT_TIMESTAMP_FORMAT).to_string() } @@ -378,6 +407,10 @@ pub fn sanitize_product_event( let timestamp = match obj.get("timestamp") { None | Some(serde_json::Value::Null) => received_at, Some(serde_json::Value::String(raw)) => match parse_product_event_timestamp(raw) { + // Parseable but implausible is dropped like unparseable: clamping to + // the window edge would pile the whole replay onto one partition and + // report every event at a time it did not happen. + Some(parsed) if !product_event_timestamp_in_range(parsed, received_at) => return false, Some(parsed) => parsed, None => return false, }, @@ -742,6 +775,38 @@ mod tests { } } + #[test] + fn product_event_timestamp_outside_the_window_is_dropped() { + // Inside: a buffered backend flush from last week, the far edge of the + // backdate window, and a clock a few hours ahead. + for ok in [ + serde_json::json!("2026-08-10T12:00:00Z"), + serde_json::json!("2026-07-19T12:00:00Z"), + serde_json::json!("2026-08-18T06:00:00Z"), + ] { + let mut obj = product_event(serde_json::json!({ "name": "e", "timestamp": ok })); + assert!( + sanitize_product_event(&mut obj, received_at()), + "timestamp {ok} should be kept" + ); + } + + // Outside: a five-year historical replay (one partition per day) and a + // far-future stamp the 365-day TTL would never reach. + for bad in [ + serde_json::json!("2019-01-01T00:00:00Z"), + serde_json::json!("2026-07-17T11:59:59Z"), + serde_json::json!("2026-08-19T00:00:00Z"), + serde_json::json!("2999-01-01T00:00:00Z"), + ] { + let mut obj = product_event(serde_json::json!({ "name": "e", "timestamp": bad })); + assert!( + !sanitize_product_event(&mut obj, received_at()), + "timestamp {bad} should be dropped" + ); + } + } + #[test] fn product_event_url_derives_host_and_page_path() { let mut obj = product_event(serde_json::json!({ diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index fff95fb7c..b0a9dcab3 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -24,8 +24,10 @@ as a real funnel in the product rather than a hand-written `run_sql`. identity; older builds keep writing (all new columns default). 6. **Billing**: `bun run --cwd apps/api atmn push` (no package script exists; `atmn` is an `apps/api` dependency) so the `product_events` feature and its `startup` plan item exist in - Autumn before the gateway starts reserving against it. Until pushed, `balances.check` for an - unknown feature fails open and usage is still tracked, so nothing is rejected — just unbilled. + Autumn before the gateway starts reserving against it. Until pushed, Autumn answers + `allowed: false` for the unknown feature — a real denial, not an error, so `is_allowed`'s + fail-open does NOT rescue it. Neither event path rejects on `product_events` for exactly that + reason: usage is recorded fail-open and nothing is dropped — just unbilled. ### Billing @@ -33,11 +35,11 @@ Product events are their own metered Autumn feature, `product_events` (unit = on separate from `browser_sessions`. The ingest gateway meters it on two paths, both with the same reserve → WAL enqueue → confirm/release shape as session starts (`metered_enqueue` in `apps/ingest/src/main.rs`): (1) `POST /v1/events` reserves the number of rows that survived -`sanitize_product_event`, and its entitlement gate is `product_events`; (2) `POST /v1/sessionEvents` -reserves the number of `type == "custom"` rows in the batch (a browser `track()` call is the same -unit as a server-side event) but keeps its entitlement REJECTION on `browser_sessions`, so an -exhausted product-events allowance bills usage-based overage instead of 402-ing a whole session -transcript. Automatic session events (clicks, navigations, ...) stay unmetered. **Beta pricing +`sanitize_product_event`; (2) `POST /v1/sessionEvents` reserves the number of `type == "custom"` +rows in the batch (a browser `track()` call is the same unit as a server-side event) and keeps its +entitlement REJECTION on `browser_sessions`. **Neither path rejects on `product_events`**: an +exhausted (or un-provisioned) product-events allowance bills usage-based overage instead of 402-ing +a whole session transcript or a backend's buffered batch. Automatic session events (clicks, navigations, ...) stay unmetered. **Beta pricing (2026-08-17): the `startup` item is `unlimited: true` with no price** — usage is tracked in Autumn (and shown on the billing page as "Unlimited · free during beta") so real volumes are known before a price is set. To start charging, swap `unlimited` for an `included` allowance + `price` in diff --git a/lib/clickhouse-builder/src/ch/functions/aggregate.ts b/lib/clickhouse-builder/src/ch/functions/aggregate.ts index ffbe7e59f..5f36caf46 100644 --- a/lib/clickhouse-builder/src/ch/functions/aggregate.ts +++ b/lib/clickhouse-builder/src/ch/functions/aggregate.ts @@ -102,9 +102,15 @@ export type WindowFunnelMode = "strict_order" | "strict_deduplication" | "strict * `cond1..condN` that occurred in that order within `window` of the `cond1` * event. * - * `window` is in the unit of `timestamp` — for `DateTime`/`DateTime64` columns - * that is seconds, so callers pass `windowSeconds`. Ordering within a group - * happens inside the aggregate; no `ORDER BY` is needed on the input. + * `window` is in the unit of `timestamp`, whatever that unit happens to be — + * seconds for a `Date`/`DateTime` column, but ClickHouse rejects `DateTime64` + * outright, so a sub-second-precision column has to be projected to an integer + * first and `window` then follows THAT unit. Projecting with + * `toUInt64(toUnixTimestamp64Milli(ts))` means passing `windowSeconds * 1000`; + * passing bare seconds against a millisecond timestamp silently yields a window + * 1000x too short and a funnel that converts almost nobody past step 1. + * Ordering within a group happens inside the aggregate; no `ORDER BY` is needed + * on the input. * * Curried like {@link quantile}: the window and mode are *parameters* of the * aggregate, the timestamp and conditions are its arguments. diff --git a/packages/domain/src/clickhouse/migrations/0016_product_events.ts b/packages/domain/src/clickhouse/migrations/0016_product_events.ts index 2f3930de2..0f05f1ed6 100644 --- a/packages/domain/src/clickhouse/migrations/0016_product_events.ts +++ b/packages/domain/src/clickhouse/migrations/0016_product_events.ts @@ -84,7 +84,8 @@ export const productEventsBrowserBackfill: BackfillSpec = { * * Re-runnable by construction: views are dropped first, `product_events` * clears only `Source = 'browser'` (never a directly ingested row), and - * `identity_links` is a ReplacingMergeTree so a re-insert of a pair is a no-op. + * `identity_links` is an AggregatingMergeTree over `min(FirstSeen)`, so a + * re-insert of a pair collapses back to the same earliest-sighting row. * * **BYO ClickHouse only.** Managed orgs get `product_events_mv` / * `identity_links_mv` via `tinybird deploy` from `materializations.ts`, and the @@ -128,13 +129,17 @@ ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) TTL toDate(Timestamp) + INTERVAL 365 DAY`, + // AggregatingMergeTree, not Replacing: the reader ranks a visitor's linked + // users by `FirstSeen`, and a Replacing merge with no version column keeps + // an arbitrary duplicate, so that ranking would flip as merges land. `min` + // makes the collapse keep the earliest sighting instead. `CREATE TABLE IF NOT EXISTS identity_links ( OrgId LowCardinality(String), VisitorId String, UserId String, - FirstSeen DateTime64(9) + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) ) -ENGINE = ReplacingMergeTree +ENGINE = AggregatingMergeTree PARTITION BY tuple() ORDER BY (OrgId, VisitorId, UserId) TTL toDate(FirstSeen) + INTERVAL 365 DAY`, diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index e6caeeafc..6e2a26818 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" as const +export const projectRevision = "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -11,7 +11,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL Minute + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS identity_links (\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen DateTime64(9)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY tuple()\nORDER BY (OrgId, VisitorId, UserId)\nTTL toDate(FirstSeen) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS identity_links (\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen SimpleAggregateFunction(min, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY tuple()\nORDER BY (OrgId, VisitorId, UserId)\nTTL toDate(FirstSeen) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS logs_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS metric_catalog (\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour)\nTTL Hour + INTERVAL 90 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 9de89811e..e24702533 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "bddca651205d7821d426fb1288b9c56c413ea09420bbac8461f23f6b18ad0b04" as const +export const projectRevision = "71718ac94d7311ca634b1c9e13ff5fecd517968b5a6d9a2854cd6f28eea336ab" as const export const datasources = [ { @@ -42,7 +42,7 @@ export const datasources = [ { name: "identity_links", content: - 'DESCRIPTION >\n Visitor→user identity links, one row per (VisitorId, UserId) pair observed on a session_replays row with both set. Stitches anonymous and identified product_events into one person for funnels.\n\nSCHEMA >\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen DateTime64(9)\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "tuple()"\nENGINE_SORTING_KEY "OrgId, VisitorId, UserId"\nENGINE_TTL "toDate(FirstSeen) + INTERVAL 365 DAY"', + 'DESCRIPTION >\n Visitor→user identity links, one row per (VisitorId, UserId) pair observed on a session_replays row with both set. Stitches anonymous and identified product_events into one person for funnels.\n\nSCHEMA >\n OrgId LowCardinality(String),\n VisitorId String,\n UserId String,\n FirstSeen SimpleAggregateFunction(min, DateTime64(9))\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "tuple()"\nENGINE_SORTING_KEY "OrgId, VisitorId, UserId"\nENGINE_TTL "toDate(FirstSeen) + INTERVAL 365 DAY"', }, { name: "logs", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index fceea3167..365ce15cf 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2194,8 +2194,20 @@ export type ProductEventsRow = InferRow * only) into one row: a `product_events` row resolves its person as * `if(UserId != '', UserId, coalesce(link.UserId, VisitorId))`. * - * ReplacingMergeTree keyed on the pair, so re-observing it is a no-op; the - * lowest `FirstSeen` wins on merge because the reader takes `min()`. + * AggregatingMergeTree keyed on the pair, with `FirstSeen` as + * `SimpleAggregateFunction(min, …)`, so re-observing a pair collapses to the + * EARLIEST sighting. That engine choice is load-bearing, not tidiness: the + * reader ranks a visitor's users by `FirstSeen` to pick the one they became + * first, and under a plain ReplacingMergeTree (no version column) a merge keeps + * an arbitrary duplicate — commonly the newest. A visitor linked to A in + * January and again in March, and to B in February, then answers "A" until the + * merge lands and "B" afterwards, moving funnel counts with merge timing rather + * than with the data. Read-time `min()` cannot repair that: by then the January + * row is gone. The merge itself has to keep the minimum. + * + * Readers still aggregate `min(FirstSeen)` per pair — unmerged parts hold + * several rows — and only then rank; see `identityLinksByVisitor` in + * `@maple/query-engine`'s `ch/queries/product-events.ts`. */ export const identityLinks = defineDatasource("identity_links", { description: @@ -2205,13 +2217,14 @@ export const identityLinks = defineDatasource("identity_links", { OrgId: t.string().lowCardinality(), VisitorId: t.string(), UserId: t.string(), - FirstSeen: t.dateTime64(9), + FirstSeen: t.simpleAggregateFunction("min", t.dateTime64(9)), }, - engine: engine.replacingMergeTree({ + engine: engine.aggregatingMergeTree({ partitionKey: "tuple()", sortingKey: ["OrgId", "VisitorId", "UserId"], - // A pair not re-observed for a year is dead weight; the reader takes the - // most recent link anyway. + // A pair not re-observed for a year is dead weight. Keyed off the pair's + // first sighting, which `min` now makes stable, so the TTL of a link does + // not move every time the visitor signs in again. ttl: "toDate(FirstSeen) + INTERVAL 365 DAY", }), }) diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 92db8c739..0210833e5 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -649,7 +649,7 @@ SELECT ORDER BY group ASC, step ASC FORMAT JSON --- builder:product-events:productEventsFunnelBreakdownQuery:session-dimension [5df587f2] +-- builder:product-events:productEventsFunnelBreakdownQuery:session-dimension [f52bc5e1] SELECT group AS group, arrayJoin([1, 2, 3]) AS step, @@ -673,8 +673,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId LEFT JOIN (SELECT SessionId AS SessionId, @@ -696,7 +701,7 @@ SELECT ORDER BY group ASC, step ASC FORMAT JSON --- builder:product-events:productEventsFunnelQuery:person [5c76c8ff] +-- builder:product-events:productEventsFunnelQuery:person [41bb75cc] SELECT arrayJoin([1, 2, 3]) AS step, arrayElement(counts, step) AS count @@ -715,8 +720,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId WHERE e.OrgId = 'org_sql_catalog' AND e.Timestamp >= '2026-01-01 10:30:00' @@ -752,7 +762,7 @@ SELECT ORDER BY step ASC FORMAT JSON --- builder:product-events:productEventsFunnelQuery:session-step-filtered [2502c451] +-- builder:product-events:productEventsFunnelQuery:session-step-filtered [bbc7eaed] SELECT arrayJoin([1, 2, 3, 4]) AS step, arrayElement(counts, step) AS count @@ -773,8 +783,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId WHERE s.OrgId = 'org_sql_catalog' AND s.StartTime >= '2026-01-01 10:30:00' @@ -787,8 +802,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId WHERE s.OrgId = 'org_sql_catalog' AND s.StartTime >= '2026-01-01 10:30:00' @@ -826,8 +846,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId WHERE e.OrgId = 'org_sql_catalog' AND e.Timestamp >= '2026-01-01 10:30:00' @@ -840,8 +865,13 @@ SELECT LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId + FROM (SELECT + VisitorId AS VisitorId, + UserId AS UserId, + min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_sql_catalog' + GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON s.VisitorId = link.VisitorId WHERE s.OrgId = 'org_sql_catalog' AND s.StartTime >= '2026-01-01 10:30:00' diff --git a/packages/query-engine/src/ch/queries/product-events.test.ts b/packages/query-engine/src/ch/queries/product-events.test.ts index 831359948..26f2bf539 100644 --- a/packages/query-engine/src/ch/queries/product-events.test.ts +++ b/packages/query-engine/src/ch/queries/product-events.test.ts @@ -99,8 +99,12 @@ describe("productEventsFunnelQuery", () => { productEventsFunnelQuery({ steps: STEPS, keyBy: "person", windowSeconds: 60 }), params, ) + // `min(FirstSeen)` per pair BEFORE the argMin: identity_links holds one + // row per sighting until the AggregatingMergeTree merge collapses them, + // so ranking a visitor's users on a raw FirstSeen would count arbitrary + // duplicates until parts merge. expect(oneLine(sql)).toContain( - "LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId FROM identity_links WHERE OrgId = 'org_1' GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId", + "LEFT JOIN (SELECT VisitorId AS VisitorId, argMin(UserId, FirstSeen) AS UserId FROM (SELECT VisitorId AS VisitorId, UserId AS UserId, min(FirstSeen) AS FirstSeen FROM identity_links WHERE OrgId = 'org_1' GROUP BY VisitorId, UserId) AS pair_links GROUP BY VisitorId) AS link ON e.VisitorId = link.VisitorId", ) expect(sql).toContain( "multiIf(e.UserId != '', e.UserId, coalesce(link.UserId, '') != '', coalesce(link.UserId, ''), e.VisitorId) AS key", diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index 1b97daf53..ec31f856e 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -221,16 +221,36 @@ type LinkAccessor = { readonly UserId: CH.Expr } | ColumnAccessor const LINK_ALIAS = "link" /** - * `identity_links` collapsed to one linked user per visitor. ReplacingMergeTree - * may still hold several rows per (visitor, user) and a visitor may have been - * linked to more than one user; `argMin(UserId, FirstSeen)` picks the user the + * `identity_links` collapsed to one linked user per visitor: the user the * visitor became *first*, which is the identity that anonymous rows * chronologically precede — the one a conversion funnel wants. + * + * TWO aggregations, and the inner one is load-bearing. `identity_links` holds + * one row per (visitor, user) SIGHTING until a merge collapses them, so a pair + * seen across several sessions has several rows with different `FirstSeen`. + * `min(FirstSeen)` per pair reduces that to the pair's first sighting, and only + * then does the outer `argMin` pick which user the visitor became first. + * + * The engine is the other half of this: `identity_links` is an + * AggregatingMergeTree whose `FirstSeen` collapses under `min`, so the merge + * keeps that same earliest value. A plain ReplacingMergeTree (no version + * column) keeps an arbitrary duplicate instead — usually the newest — and no + * amount of read-time aggregation recovers the row it dropped, so a visitor + * linked to A in January and again in March, and to B in February, would answer + * "A" until the merge landed and "B" after. Neither half works alone. */ function identityLinksByVisitor() { - return from(IdentityLinks) - .select(($) => ({ VisitorId: $.VisitorId, UserId: CH.argMin($.UserId, $.FirstSeen) })) + const firstSeenPerPair = from(IdentityLinks) + .select(($) => ({ + VisitorId: $.VisitorId, + UserId: $.UserId, + FirstSeen: CH.min_($.FirstSeen), + })) .where(($) => [$.OrgId.eq(param.string("orgId"))]) + .groupBy("VisitorId", "UserId") + + return fromQuery(firstSeenPerPair, "pair_links") + .select(($) => ({ VisitorId: $.VisitorId, UserId: CH.argMin($.UserId, $.FirstSeen) })) .groupBy("VisitorId") } diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index e798b6294..ae0d630b8 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -706,8 +706,11 @@ export const ProductEvents = table("product_events", { }) // (VisitorId, UserId) pairs observed together on a session_replays row. -// ReplacingMergeTree keyed on the pair — always aggregate (`min(FirstSeen)`) or -// semi-join; never assume one row per pair on read. +// AggregatingMergeTree keyed on the pair, `FirstSeen` collapsing under `min` — +// so a merge keeps the pair's EARLIEST sighting rather than an arbitrary one, +// which is what makes ranking a visitor's users by it stable. Unmerged parts +// still hold several rows per pair, so always aggregate (`min(FirstSeen)` per +// pair) or semi-join; never assume one row per pair on read. export const IdentityLinks = table("identity_links", { OrgId: T.string, VisitorId: T.string, From 37aa0bf473995813c43fd764daf893e0099e507a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 21 Aug 2026 19:24:22 +0200 Subject: [PATCH 15/15] fix(funnels): adapt funnel chart call to TanStack charts props post-merge --- apps/web/src/components/funnels/funnel-results.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/web/src/components/funnels/funnel-results.tsx b/apps/web/src/components/funnels/funnel-results.tsx index 4458acae3..6b5847e4b 100644 --- a/apps/web/src/components/funnels/funnel-results.tsx +++ b/apps/web/src/components/funnels/funnel-results.tsx @@ -70,11 +70,7 @@ export function FunnelResults({ labels, rows, unitNoun, waiting = false, classNa
) : ( }> - + )}