diff --git a/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md b/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md index 5f3e93972..4c0e39ed7 100644 --- a/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md +++ b/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md @@ -58,14 +58,6 @@ Per-service hosting-platform attributes for the service map's runtime-icon resol Additionally extracts (see file): `faas.name`, `sdk.type`, `process.runtime.name`. -## `error_spans_mv` - -Materializes spans with `StatusCode = 'Error'`. - -| Column | Extracted from | Line | -|---|---|---| -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:454` | - ## `error_events_mv` Unwraps the first OTel `exception` event from `EventsName` / `EventsAttributes` Maps. diff --git a/CLAUDE.md b/CLAUDE.md index 010384f92..a8e33d76d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -172,6 +172,7 @@ there is no Prometheus `/metrics` endpoint. At high QPS set `OTEL_TRACES_SAMPLER ## Docs (`docs/`) `api-v2.md` (v2 public API spec) · `sampling-throughput.md` · `persistence.md` · +`warehouse-rollups.md` (MV/rollup tiering contract — read before adding a materialized view) · `sst-fork-workflow.md` · `local-mode.md` (single-binary CLI + embedded chDB) · `tinybird-pr-branches.md` · `otel-spec/` (OTel spec map @ v1.58.0 — start at its README). diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 1f154f0ff..0992a9261 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,4 +1,4 @@ -import { formatWarehouseDateTime } from "@maple/query-engine" +import { formatWarehouseDateTime, snapAlertWindowEndMs } from "@maple/query-engine" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -502,7 +502,10 @@ export class AlertsService extends Context.Service { yield* Effect.annotateCurrentSpan({ orgId, "maple.alert.rule_id": rule.id }) - const endMs = yield* now + // Snapped, not raw `now`: excludes the still-filling current minute and + // lets rules over the same signal share one `qe-evaluate` entry within + // a tick. See `snapAlertWindowEndMs`. + const endMs = snapAlertWindowEndMs(yield* now) const startMs = endMs - rule.windowMinutes * 60_000 const plan = rule.compiledPlan const source = yield* planEvaluateSource(plan, rule.windowMinutes) diff --git a/apps/api/src/services/warehouse/QueryEngineService.ts b/apps/api/src/services/warehouse/QueryEngineService.ts index 33bf3a62d..a89ba0f96 100644 --- a/apps/api/src/services/warehouse/QueryEngineService.ts +++ b/apps/api/src/services/warehouse/QueryEngineService.ts @@ -352,9 +352,14 @@ export class QueryEngineService extends Context.Service { expect(names).toContain("logs") expect(names).toContain("traces") expect(names).toContain("service_overview_spans") - expect(names).toContain("error_spans") expect(names).toContain("metrics_gauge") }) diff --git a/apps/api/src/services/warehouse/warehouse-catalog.ts b/apps/api/src/services/warehouse/warehouse-catalog.ts index c51f9f215..b215844f8 100644 --- a/apps/api/src/services/warehouse/warehouse-catalog.ts +++ b/apps/api/src/services/warehouse/warehouse-catalog.ts @@ -35,11 +35,6 @@ const TABLE_NOTES: Record> = { "`StatusCode` is Title Case: 'Ok', 'Error', 'Unset'.", "Does NOT include `SpanAttributes`/`ResourceAttributes` — query `traces` if you need attribute access.", ], - error_spans: [ - "Pre-filtered to `StatusCode = 'Error'`. Use this instead of `traces WHERE StatusCode = 'Error'`.", - "`Duration` is NANOSECONDS.", - "`DeploymentEnv` is pre-extracted from ResourceAttributes['deployment.environment'].", - ], error_events: [ "Per-error-occurrence rows with the OTel `exception` event unwrapped — surfaces `ExceptionType`, `ExceptionMessage`, `Stacktrace`, and a stable `FingerprintHash` for grouping.", "Use `FingerprintHash` to group occurrences into issues; `(OrgId, FingerprintHash, Timestamp)` is the sort key.", diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 01f47a706..7db633f51 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -79,4 +79,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "60908c2e8307e24885227d4553916eef64df7f9b23abec23b5697cfea0d84d94", projectRevision: "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", }), + Object.freeze({ + version: 9, + fingerprint: "2516215f22b41a63", + digest: "2516215f22b41a636b3186d0b293a0a6276e4bb85004efd3994b80867696a469", + manifestDigest: "f1bdef1ca3dcb073fc3cda998c145f64fd9a4a4f174543e7bc0405867bedf356", + projectRevision: "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 8bbe13f03..76baaad5f 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 8 as const +export const LOCAL_SCHEMA_VERSION = 9 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index f41dc937c..e46100d36 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -44,6 +44,7 @@ import { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4 import { v5ToV6ErrorEventsFingerprintHygieneModule } from "./local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene" import { v6ToV7ErrorServiceVersionModule } from "./local-store-migrations/v6-to-v7-error-service-version" import { v7ToV8AppleCrashFramesModule } from "./local-store-migrations/v7-to-v8-apple-crash-frames" +import { v8ToV9MvSweepModule } from "./local-store-migrations/v8-to-v9-mv-sweep" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -109,6 +110,7 @@ export const localStoreMigrations: ReadonlyArray = v5ToV6ErrorEventsFingerprintHygieneModule, v6ToV7ErrorServiceVersionModule, v7ToV8AppleCrashFramesModule, + v8ToV9MvSweepModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts b/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts new file mode 100644 index 000000000..84aa1a1a5 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts @@ -0,0 +1,260 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { + LOCAL_SCHEMA_V8, + LOCAL_SCHEMA_V8_MANIFEST, + LOCAL_SCHEMA_V8_SQL, + LOCAL_SCHEMA_V9, + LOCAL_SCHEMA_V9_MANIFEST, + LOCAL_SCHEMA_V9_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0008-to-0009-mv-sweep" as const + +const V8ToV9StateCodec = makeRawRowsState(MODULE_ID) + +type V8ToV9State = typeof V8ToV9StateCodec.schema.Type +type V8ToV9Progress = InstalledProgress + +const decodeState = V8ToV9StateCodec.decode +const decodeProgress = decodeInstalledProgress + +/** + * Views and tables this edge must remove before the v9 DDL installs. + * + * The bundled DDL is `CREATE ... IF NOT EXISTS` throughout, so a view whose body + * changed has to be dropped or the v8 version simply survives. `error_spans` and + * its view go further — they are gone from v9 entirely, and `assertPhysicalSchema` + * fails on LEFTOVER objects, not only missing ones (`unexpected materialized_view`). + * A clone-only edge would therefore fail verification rather than silently drift. + * + * chDB materializes views as tables, so `DROP TABLE` is the right verb for both — + * the same spelling v5 -> v6 and v7 -> v8 use. + * + * ORDER: every view precedes the table it writes into. The inverse leaves an MV + * pointing at a missing target and wedges inserts into the source table. + */ +const DROPPED_VIEWS = [ + "error_spans_mv", + "trace_detail_spans_mv", + "log_attribute_values_mv", + "metric_attribute_values_mv", + "trace_span_attribute_values_mv", + "trace_resource_attribute_values_mv", + "span_metrics_calls_hourly_mv", +] as const + +const DROPPED_TABLES = ["error_spans"] as const + +/** + * Columns removed from `trace_detail_spans`, dropped explicitly for the same + * reason the views above are: the bundled v9 DDL is `CREATE TABLE IF NOT EXISTS`, + * so on a store where the table already exists it is a no-op and the wide v8 + * table survives. Installing the narrowed schema is NOT enough — `assertPhysicalSchema` + * then fails with `unexpected column EventsTimestamp`. + */ +const DROPPED_COLUMNS: ReadonlyArray = [ + ["trace_detail_spans", "EventsTimestamp"], + ["trace_detail_spans", "EventsName"], + ["trace_detail_spans", "EventsAttributes"], +] + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V8_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V8_SQL, bootstrapSchema: false }, + ) + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V8ToV9State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * The local mirror of ClickHouse migration 0019. Four changes, one edge: + * + * - `error_spans` and its view are dropped outright. Nothing read them. + * - `trace_detail_spans` loses its three `Events*` columns, via an explicit + * `ALTER ... DROP COLUMN`. Declaring the narrowed table in the v9 DDL is not + * enough — that DDL is `CREATE TABLE IF NOT EXISTS`, so on an existing store + * it is a no-op and the wide v8 table survives verification as + * `unexpected column EventsTimestamp`. Same `IF NOT EXISTS` trap as the views. + * - The four attribute-value views gain a cardinality bound. + * - The span-metrics calls view starts matching the metric name the collector + * actually emits. + * + * Every one of these is forward-only. Rows already materialized keep whatever + * the v8 bodies produced, and the targets converge as their TTL rolls — the same + * position a deployed cluster is in right after migration 0019. Backfilling + * would mean rewriting a store this edge has just promised to clone + * byte-for-byte, and for `attribute_values_hourly` the whole point is that the + * old rows are the ones we no longer want to keep. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + for (const view of DROPPED_VIEWS) db.exec(`DROP TABLE IF EXISTS ${view}`) + for (const table of DROPPED_TABLES) db.exec(`DROP TABLE IF EXISTS ${table}`) + for (const [table, column] of DROPPED_COLUMNS) { + db.exec(`ALTER TABLE ${table} DROP COLUMN IF EXISTS ${column}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V8_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V9_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V8ToV9State, + _progress: V8ToV9Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V9_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v8 -> v9 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V9_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v8-store", + description: "Clone the stopped v8 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "sweep-materialized-views", + description: + "Drop the unread error_spans table and rebuild the trace-detail, attribute-value and span-metrics views", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v9-schema", + description: "Verify the v9 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v8 store is cloned byte-for-byte before any view is replaced.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of every replaced view is neither read nor rewritten; only view definitions and their targets change.", + }, + { + // Removed outright, and it does not come back — hence `invalidate` rather + // than one of the rebuild dispositions. + name: "error_spans", + classification: "derived", + disposition: "invalidate", + guarantee: + "Dropped permanently along with its view. It had no readers: every error query reads error_events / error_events_by_time, and the rows were reproducible from traces in any case.", + }, + { + // Narrowed, not rebuilt: the three Events columns had no readers. + name: "trace_detail_spans", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Rows are preserved; the unread EventsTimestamp/EventsName/EventsAttributes columns are gone from the v9 table and converge as the 30-day window rolls.", + preservationInterval: "trace retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 30, + }, + { + name: "attribute_values_hourly", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows are preserved untouched; the cardinality bound applies to values materialized after the migration and the unbounded history ages out with the 90-day TTL.", + preservationInterval: "attribute retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, + { + // Empty on every existing store: the view never matched a metric name. + name: "span_metrics_calls_hourly", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "The rollup was empty because its view matched a name nothing emits; it begins filling from the corrected view and is complete within the 90-day horizon.", + preservationInterval: "metric rollup horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, +] + +export const v8ToV9MvSweepModule: LocalStoreMigrationModule = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Drop the unread error_spans table and rebuild the trace-detail, attribute-value and span-metrics views", + from: LOCAL_SCHEMA_V8, + to: LOCAL_SCHEMA_V9, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index dcf566f1b..cd4118c2e 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -7,6 +7,7 @@ import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import schemaV7Sql from "./schema/local-schema-v7.sql" with { type: "text" } import schemaV8Sql from "./schema/local-schema-v8.sql" with { type: "text" } +import schemaV9Sql from "./schema/local-schema-v9.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -30,7 +31,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" + "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -59,6 +60,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV6Sql, schemaV7Sql, schemaV8Sql, + schemaV9Sql, ] export interface LocalSchemaSnapshot { @@ -107,6 +109,9 @@ export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = snapshotAt(7).manifestDigest export const LOCAL_SCHEMA_V8_SQL = snapshotAt(8).sql export const LOCAL_SCHEMA_V8_MANIFEST = snapshotAt(8).manifest export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = snapshotAt(8).manifestDigest +export const LOCAL_SCHEMA_V9_SQL = snapshotAt(9).sql +export const LOCAL_SCHEMA_V9_MANIFEST = snapshotAt(9).manifest +export const LOCAL_SCHEMA_V9_MANIFEST_DIGEST = snapshotAt(9).manifestDigest export interface LocalSchemaIdentity { readonly version: number @@ -145,6 +150,7 @@ export const LOCAL_SCHEMA_V5 = identityAt(5) export const LOCAL_SCHEMA_V6 = identityAt(6) export const LOCAL_SCHEMA_V7 = identityAt(7) export const LOCAL_SCHEMA_V8 = identityAt(8) +export const LOCAL_SCHEMA_V9 = identityAt(9) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index a4ab5e71b..ba5d601cc 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", + "projectRevision": "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v9.sql b/apps/cli/src/server/schema/local-schema-v9.sql new file mode 100644 index 000000000..26a2f3bbc --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v9.sql @@ -0,0 +1,1833 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979 +-- localSchemaVersion: 9 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 60ba1b4b1..26a2f3bbc 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534 --- localSchemaVersion: 8 +-- projectRevision: 3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979 +-- localSchemaVersion: 9 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -120,22 +120,6 @@ PARTITION BY toYYYYMM(Minute) ORDER BY (OrgId, Minute, FingerprintHash) TTL Minute + INTERVAL 90 DAY; -CREATE TABLE IF NOT EXISTS error_spans ( - OrgId LowCardinality(String), - Timestamp DateTime, - TraceId String, - SpanId String, - ParentSpanId String DEFAULT '__unset__', - ServiceName LowCardinality(String), - StatusMessage String, - Duration UInt64, - DeploymentEnv LowCardinality(String) -) -ENGINE = MergeTree -PARTITION BY toDate(Timestamp) -ORDER BY (OrgId, ServiceName, Timestamp) -TTL Timestamp + INTERVAL 90 DAY; - CREATE TABLE IF NOT EXISTS logs ( OrgId LowCardinality(String), Timestamp DateTime64(9), @@ -737,10 +721,7 @@ CREATE TABLE IF NOT EXISTS trace_detail_spans ( StatusCode LowCardinality(String), StatusMessage String, SpanAttributes Map(LowCardinality(String), String), - ResourceAttributes Map(LowCardinality(String), String), - EventsTimestamp Array(DateTime64(9)), - EventsName Array(LowCardinality(String)), - EventsAttributes Array(Map(LowCardinality(String), String)) + ResourceAttributes Map(LowCardinality(String), String) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1152,20 +1133,6 @@ SELECT FROM error_events GROUP BY OrgId, Minute, FingerprintHash; -CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS -SELECT - OrgId, - toDateTime(Timestamp) AS Timestamp, - TraceId, - SpanId, - ParentSpanId, - ServiceName, - StatusMessage, - Duration, - ResourceAttributes['deployment.environment'] AS DeploymentEnv - FROM traces - WHERE StatusCode = 'Error'; - CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS SELECT OrgId, @@ -1190,6 +1157,11 @@ SELECT mapKeys(LogAttributes) AS AttributeKey, mapValues(LogAttributes) AS AttributeValue WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS @@ -1229,6 +1201,11 @@ SELECT mapKeys(Attributes) AS AttributeKey, mapValues(Attributes) AS AttributeValue WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS @@ -1699,7 +1676,14 @@ SELECT StartTimeUnix, argMaxState(Value, TimeUnix) AS LastValue FROM metrics_sum - WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS @@ -1716,10 +1700,7 @@ SELECT StatusCode, StatusMessage, SpanAttributes, - ResourceAttributes, - EventsTimestamp, - EventsName, - EventsAttributes + ResourceAttributes FROM traces; CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS @@ -1779,6 +1760,11 @@ SELECT mapKeys(ResourceAttributes) AS AttributeKey, mapValues(ResourceAttributes) AS AttributeValue WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS @@ -1805,6 +1791,11 @@ SELECT mapKeys(SpanAttributes) AS AttributeKey, mapValues(SpanAttributes) AS AttributeValue WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index fba79310a..53a662554 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -20,6 +20,7 @@ import { LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V7, LOCAL_SCHEMA_V8, + LOCAL_SCHEMA_V9, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -61,16 +62,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v8 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("51081e951066442a") - expect(SCHEMA_DIGEST).toBe("51081e951066442a8e5b53df2c4bdda933edd20fc89132a54ed9b4dbb7e55a05") + it("matches the generated v9 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("2516215f22b41a63") + expect(SCHEMA_DIGEST).toBe("2516215f22b41a636b3186d0b293a0a6276e4bb85004efd3994b80867696a469") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(8) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V8) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(9) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V9) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -125,15 +126,23 @@ describe("current local schema identity", () => { expect(minutelyView?.definition).toContain("FROM traces") expect(minutelyView?.definition).not.toContain("FROM service_overview_minutely") const v5Names = new Set(LOCAL_SCHEMA_V5_MANIFEST.objects.map((object) => object.name)) + const currentNames = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) expect([...v5Names].filter((name) => !v4Names.has(name))).toEqual([ "service_overview_minutely", "service_overview_minutely_mv", ]) - // v6 adds and removes nothing: it only replaces two materialized-view - // bodies, so the object set is identical to v5 and the manifest digest - // differs solely through those two definitions. - expect(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)).toEqual([...v5Names]) + // v6, v7 and v8 add and remove nothing: they only replace materialized-view + // bodies, so their object set is identical to v5 and the manifest digest + // differs solely through those definitions. v9 is the first edge to remove + // objects — `error_spans` and its view, which no query ever read. Asserted + // as an exact set difference rather than a relaxed check, so a future edge + // still cannot add or drop an object unnoticed. + expect([...v5Names].filter((name) => !currentNames.has(name))).toEqual([ + "error_spans", + "error_spans_mv", + ]) + expect([...currentNames].filter((name) => !v5Names.has(name))).toEqual([]) const errorEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "error_events_mv", ) @@ -178,6 +187,7 @@ describe("local migration registry", () => { "local-0005-to-0006-error-events-fingerprint-hygiene", "local-0006-to-0007-error-service-version", "local-0007-to-0008-apple-crash-frames", + "local-0008-to-0009-mv-sweep", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -224,7 +234,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 9, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 10, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 0c6444e38..8967a6159 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 8 and .schema == "51081e951066442a"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 9 and .schema == "2516215f22b41a63"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index e3eae1755..be0a5cf86 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534"; +pub const PROJECT_REVISION: &str = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/docs/self-hosted-clickhouse.md b/docs/self-hosted-clickhouse.md index c430ae83c..f6fee6fe6 100644 --- a/docs/self-hosted-clickhouse.md +++ b/docs/self-hosted-clickhouse.md @@ -106,18 +106,37 @@ SELECT version, applied_at, description FROM _maple_schema_migrations ORDER BY v ## What gets created -On a clean install, migration 0001 creates **20 tables** (datasources) and **22 materialized views**: - -- **Direct-ingest tables**: `traces`, `logs`, `metrics_sum`, `metrics_gauge`, `metrics_histogram`, `metrics_exponential_histogram`, `alert_checks` -- **MV-populated tables**: `service_usage`, `service_map_spans`, `service_map_children`, `service_map_edges_hourly`, `service_overview_spans`, `error_spans`, `error_events`, `trace_list_mv`, `trace_detail_spans`, `attribute_keys_hourly`, `attribute_values_hourly`, `traces_aggregates_hourly`, `logs_aggregates_hourly` -- **Materialized views**: 22 MVs that fan out from the direct-ingest tables to populate the MV-populated tables +On a clean install, migration 0001 creates **37 tables** (datasources) and **39 materialized views**. +Migration 0001 re-exports the *generated* snapshot, so these counts track +`datasources.ts` / `materializations.ts` — regenerate with `bun run clickhouse:schema` +and `bun run tinybird:manifest` after editing either, or CI's drift gate fails. + +- **Direct-ingest tables** (12): `alert_checks`, `logs`, `metrics_exponential_histogram`, + `metrics_gauge`, `metrics_histogram`, `metrics_sum`, `service_address_resolutions_hourly`, + `service_map_edges_hourly_ingest`, `session_events`, `session_replay_events`, + `session_replays`, `traces` +- **MV-populated tables** (25): `attribute_keys_hourly`, `attribute_values_hourly`, + `error_events`, `error_events_by_time`, `error_fingerprints_minutely`, + `logs_aggregates_hourly`, `metric_catalog`, `service_external_edges_hourly`, + `service_map_children`, `service_map_db_edges_hourly`, + `service_map_db_query_shapes_hourly`, `service_map_edges_hourly`, `service_map_spans`, + `service_operations_hourly`, `service_operations_minutely`, `service_overview_hourly`, + `service_overview_minutely`, `service_overview_spans`, `service_platforms_hourly`, + `service_usage`, `span_metrics_calls_hourly`, `trace_detail_spans`, `trace_list_mv`, + `traces_aggregates_hourly`, `web_events` +- **Materialized views** (39): fan out from the direct-ingest tables to populate the + MV-populated tables. Several targets are fed by more than one MV — `service_usage` by + six, `attribute_values_hourly` / `attribute_keys_hourly` / `metric_catalog` by four each. + +See [`warehouse-rollups.md`](warehouse-rollups.md) for when a materialized view is the +right answer and which tier a query should read. Every table is partitioned by date and carries a TTL, tiered by how raw the data is: | Retention | Tables | | ------------ | ------------------------------------------------------------------------------------------------------------------------------ | | **30 days** | `traces`, `trace_detail_spans`, `logs`, `service_map_spans`, `service_map_children`, `service_overview_spans`, `trace_list_mv` | -| **90 days** | `error_spans`, `error_events`, `error_events_by_time`, `metrics_*`, `attribute_*_hourly`, `metric_catalog` | +| **90 days** | `error_events`, `error_events_by_time`, `metrics_*`, `attribute_*_hourly`, `metric_catalog` | | **365 days** | hourly rollups (`*_hourly`), `service_usage`, `alert_checks` | Adjust by writing a follow-up migration if your retention requirements differ. diff --git a/docs/warehouse-rollups.md b/docs/warehouse-rollups.md new file mode 100644 index 000000000..4f79015b4 --- /dev/null +++ b/docs/warehouse-rollups.md @@ -0,0 +1,140 @@ +# Warehouse rollups and materialized views + +We have 39 materialized views across 37 datasources. They accreted one product feature at a +time, and for a long time nobody could answer "should this be an MV?" without re-deriving it +from scratch. This is that answer. + +Definitions live in `packages/domain/src/tinybird/materializations.ts` (MVs) and +`datasources.ts` (targets). Everything else — the ClickHouse DDL snapshot, `local-schema.sql` +for the embedded chDB engine, the Rust insert mappings — is generated from those two files. + +--- + +## The tier ladder + +| Tier | Grain | TTL | Answers | +|---|---|---|---| +| **raw** | one row per span/log/point | 30d | "show me this exact trace / these log lines" — anything needing attributes, span names, or a specific id | +| **minutely** | pre-aggregated per minute | 90d | sub-hour timeseries and alert evaluation, where a per-span scan is the only alternative | +| **hourly** | pre-aggregated per hour | 365d | dashboards and trends over days-to-a-year | + +**A query must read the coarsest tier that can answer it.** The routing guards +(`canUseAnnualServiceOverview`, `canUseTracesAggregatesMv`, `canUseServiceOverviewMv`, +`canUseLogsAggregatesHourly`) exist to enforce that, and each one names the tier it unlocks. + +Rollup routes union a **raw edge** with a **rollup interior**: the rollup answers whole +buckets, and the raw table covers the partial buckets at each end of the window. Getting the +edge grain wrong double-counts every span in the first and last partial bucket, and the SQL +looks perfectly reasonable while it does — see the comment on `edgeGrain` in +`queries/traces.ts`. + +--- + +## When a materialized view is justified + +Three shapes, all already in the codebase. If a proposed MV is not one of these, it is a copy +of a table we already have. + +1. **Re-sort for point lookup.** `trace_detail_spans` is a 1:1 copy of `traces` ordered by + `(OrgId, TraceId, SpanId)` instead of `(OrgId, ServiceName, Timestamp)`. That turns + "fetch this trace" from a bloom-filter scan into a primary-key lookup. It costs 82 GB — + the single most expensive object we have — and it earns it. +2. **Pre-aggregation for scans.** `*_aggregates_hourly`, `service_overview_*`, + `service_operations_*`. Trades write amplification for orders-of-magnitude less read. +3. **Filtered projection.** `error_events` keeps only `StatusCode = 'Error'` and unwraps the + exception event, so error queries never touch the Map columns of the full traces table. + +Storage is not free and the ratio is worse than it looks: `traces` is 110 GB, and its MV +descendants total roughly 116 GB. **We store traces more than twice over.** Every new MV on +`traces` adds to that. + +--- + +## Rules + +These are not style preferences. Each one is here because its absence cost something +measurable, found during the 2026-08 sweep. + +### Every MV names its consuming query + +No consumer, no MV. `error_spans` was materialized from every error span for the lifetime of +the product and **never read once** — no query in the DSL ever called `from(ErrorSpans)`. +Worse, it was still described to LLM agents in `warehouse-catalog.ts` as "use this instead of +`traces WHERE StatusCode = 'Error'`", so `run_sql` agents were actively steered onto it. + +### Any MV that fans out over unbounded values must carry a cardinality bound + +`ARRAY JOIN` over an attribute map turns every distinct value into its own row. With +`AttributeValue != ''` as its only filter, `attribute_values_hourly` reached **1.59 billion +rows / 12.3 GB** to serve an autocomplete dropdown read a couple hundred times a week. + +Bound it by measuring what is actually in the table, not by guessing. The intuitive rules — +a length cap and a denylist of id-ish key names — would have missed the top two keys, which +were `idle_ns` and `busy_ns`: short *numeric measurements* that together were ~70% of the +rows. See `attributeValueCardinalityBound` for the rules that resulted and why. + +### Keep the write filter and the read guard in sync, and test that they agree + +`span_metrics_calls_hourly` sat at **0 rows for its entire existence**. The MV filtered +`MetricName IN ('span.metrics.calls', 'calls')`; the collector emits the counter namespaced +by its pipeline, as `traces.span.metrics.calls`. Both the write filter and the read fast-path +missed it, so they agreed with each other and disagreed with reality — and every read fell +back to a raw scan measured at ~7s p95. + +A rollup whose target is empty is indistinguishable from a rollup nobody queries. Neither the +schema lint nor the SQL catalog can catch this; only looking at row counts can. + +### A routing guard must be tested on the tier it selects, not on a table name + +`canUseAnnualServiceOverview` required `allMetrics === true`. Alert evaluation +(`computeAlertBuckets`) never sets it, so every alert fell through to a flat scan of the +325M-row per-span `service_overview_spans` — **165k scans per 3 days** — while the 2.2M-row +minutely rollup built for exactly that shape sat unreachable. + +`sql-catalog.test.ts` already asserts every routing predicate is exercised both ways, and it +passed throughout. The gap was that the test guarding this route asserted +`sql.toContain("FROM service_overview_spans")` — and the tiered union reads that table too, +as its raw edge. The assertion could not tell the two routes apart. **Assert the tier that +distinguishes the branch** (`FROM service_overview_minutely`), and include a fixture shaped +like a real alert rule, not just a dashboard query. + +--- + +## Before adding or changing an MV + +1. Name the query that will read it, and the tier it belongs to. +2. If it fans out over values, decide the cardinality bound first. +3. Add a routing fixture that pins the **tier**, both ways. +4. `bun run clickhouse:schema` and `bun run tinybird:manifest` to regenerate. +5. Write the numbered migration in `packages/domain/src/clickhouse/migrations/` — + **`DROP VIEW` before `DROP TABLE`, always**. The inverse leaves an MV pointing at a + missing target and wedges ingest with `Code: 60 UNKNOWN_TABLE`. +6. Set `requiredForIngest: false` unless the Rust gateway writes the table directly. + Bumping `clickHouseSchemaVersion` un-readies ingest routing for every BYO-ClickHouse org. +7. Bump `LOCAL_SCHEMA_VERSION`, retain the snapshot, and add the local-store migration edge — + including explicit drops, because `assertPhysicalSchema` fails on leftover objects too. + +Because step 5–7 are expensive, **batch removals into one migration** rather than shipping +them one at a time. Migration `0019_mv_sweep` is the worked example. + +--- + +## Known-unresolved + +Recorded so the next sweep does not re-derive them. + +- **`alertRawQuery` — 173,000 s/week**, the single largest consumer of warehouse time. It is + user-authored SQL against raw tables, so no MV can see it and no routing guard applies. + Needs its own design: a rollup users can target, or per-rule result caching. +- **`trace_detail_spans` TTL.** 82 GB, and TTL is the only remaining lever — column narrowing + recovered 1.19 GB (1.5%) and the rest is `SpanAttributes` (31 GB) and the incompressible + `SpanId` (17 GB). A product decision about the trace-drilldown window. +- **`service_map_spans` + `service_map_children`** — 18.6 GB of intermediate producing a + 223 MB hourly rollup. Suspicious amplification, but the recent-window branch reads them + directly. +- **`metrics_exponential_histogram` is empty.** Not dead: ingest writes it + (`apps/ingest/src/telemetry.rs`), and it is empty only because no SDK currently exports + exponential histograms. Leave it and its two MVs alone. +- **Naming drift** — `trace_list_mv_mv`, and `serviceMapDbQuerySignaturesHourlyMv` producing + `service_map_db_query_shapes_hourly_mv`. Cosmetic; renaming costs a full migration plus a + local schema version, which is not worth spending on aesthetics. diff --git a/lib/clickhouse-builder/src/ch/query.ts b/lib/clickhouse-builder/src/ch/query.ts index 67791cd64..70f0edfdf 100644 --- a/lib/clickhouse-builder/src/ch/query.ts +++ b/lib/clickhouse-builder/src/ch/query.ts @@ -18,7 +18,7 @@ // // Type-safe joins: // CH.from(Traces) -// .innerJoin(ErrorSpans, "e", (main, e) => main.TraceId.eq(e.TraceId)) +// .innerJoin(Events, "e", (main, e) => main.TraceId.eq(e.TraceId)) // .select($ => ({ // traceId: $.TraceId, // errorType: $.e.ErrorType, diff --git a/packages/domain/src/clickhouse/migrations/0019_mv_sweep.ts b/packages/domain/src/clickhouse/migrations/0019_mv_sweep.ts new file mode 100644 index 000000000..2c82559b8 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0019_mv_sweep.ts @@ -0,0 +1,78 @@ +/** + * Migration 0019 — materialized-view sweep. + * + * Four independent corrections, batched into one migration because each one on + * its own would cost a numbered ClickHouse migration AND a local-schema version + * bump with a retained snapshot and a migration edge. + * + * 1. DROP `error_spans` + its MV. Zero readers: nothing in the query DSL ever + * called `from(ErrorSpans)`. It cost 220 MB / 7.4M rows of pure write-and-store, + * and `warehouse-catalog.ts` was still advertising it to LLM agents as "use + * this instead of `traces WHERE StatusCode = 'Error'`" — so `run_sql` + * agents were actively steered onto a table no product code reads. + * + * 2. DROP the three `Events*` columns from `trace_detail_spans`. Unused by all + * nine readers (span hierarchy/detail, trace list, span search, session trace + * summaries, and the two AI-session queries). Worth 1.19 GB of the table's + * 79 GB — small, but the columns were dead weight on the largest MV we have. + * The exception-event unwrapping that DOES need them reads `traces`, not this + * table, so `error_events_mv` is unaffected. + * + * 3. Recreate the four `attribute_values_hourly` MVs with a cardinality bound. + * With `AttributeValue != ''` as their only filter the target had reached + * 1.59 BILLION rows / 12.3 GB to serve an autocomplete dropdown read a couple + * hundred times a week. Measured against production, the new predicate drops + * 93.6% of rows while dropping ZERO values for every canonical OTel key + * (`http.response.status_code`, `http.method`, `db.system`, `service.name`, + * `deployment.environment`, ...). See the note on + * `attributeValueCardinalityBound` for why the rules are what they are. + * + * 4. Recreate `span_metrics_calls_hourly_mv` matching `traces.span.metrics.calls`. + * This is a BUG FIX, not a cleanup: the MV filtered + * `MetricName IN ('span.metrics.calls', 'calls')`, but the collector emits the + * counter namespaced by its pipeline. The rollup therefore held 0 rows for its + * entire existence while ~880k rows / 2 days of the real counter flowed into + * `metrics_sum`, and every read fell back to the raw window-function scan the + * read path measures at ~7s p95. + * + * Statement ORDER is load-bearing: `DROP VIEW` always precedes `DROP TABLE`. + * The inverse leaves an MV whose `TO` target no longer exists, and ClickHouse + * fails every subsequent insert into the source table with `Code: 60 UNKNOWN_TABLE` + * — i.e. it wedges ingest. `scripts/lint-clickhouse-schema.ts` enforces the same + * invariant on the snapshot. + * + * MV bodies are frozen copies of the snapshot at the time this migration was + * written, NOT re-read from `latestSnapshotStatements`. A delta migration has to + * describe one step in history: if it rendered the live snapshot, a server at + * version 18 would jump straight to a body containing changes from migrations it + * has not applied yet. + * + * `requiredForIngest: false` — every table here is MV-populated, the Rust + * gateway writes none of them, and bumping `clickHouseSchemaVersion` would + * un-ready ingest routing for every BYO-ClickHouse org over a read-path change. + */ +export const migration_0019_mv_sweep = { + version: 19, + description: + "Drop the unread error_spans table and trace_detail_spans event columns; bound attribute_values_hourly cardinality; fix the span-metrics calls rollup metric name", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_spans_mv", + "DROP TABLE IF EXISTS error_spans", + "DROP VIEW IF EXISTS trace_detail_spans_mv", + "ALTER TABLE trace_detail_spans DROP COLUMN IF EXISTS EventsTimestamp", + "ALTER TABLE trace_detail_spans DROP COLUMN IF EXISTS EventsName", + "ALTER TABLE trace_detail_spans DROP COLUMN IF EXISTS EventsAttributes", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes\n FROM traces", + "DROP VIEW IF EXISTS log_attribute_values_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "DROP VIEW IF EXISTS metric_attribute_values_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "DROP VIEW IF EXISTS trace_span_attribute_values_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "DROP VIEW IF EXISTS trace_resource_attribute_values_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "DROP VIEW IF EXISTS span_metrics_calls_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n -- 'traces.span.metrics.calls' is the name the collector actually emits:\n -- spanmetricsconnector output is namespaced by the pipeline it is attached\n -- to. Without it this MV matched nothing and the target sat at 0 rows since\n -- it was created, while ~880k rows / 2 days of the real counter flowed past\n -- into metrics_sum and every read fell back to the raw window-function scan\n -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with\n -- SPAN_METRICS_CALLS_NAMES on the read side.\n WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 714722487..3447c2b15 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -24,6 +24,7 @@ import { import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" +import { migration_0019_mv_sweep } from "./0019_mv_sweep" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -39,12 +40,12 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ]) - expect(migrations.at(-1)).toBe(migration_0018_apple_crash_frames) - expect(latestMigrationVersion).toBe(18) - // 0010 and 0014-0018 are read-path only, so the ingest-gating version skips - // all six and stays at 13 — nothing writes `web_events`, + expect(migrations.at(-1)).toBe(migration_0019_mv_sweep) + expect(latestMigrationVersion).toBe(19) + // 0010 and 0014-0019 are read-path only, so the ingest-gating version skips + // all seven and stays at 13 — nothing writes `web_events`, // `service_overview_minutely` or `error_events` directly, and bumping it // would un-ready every BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") @@ -54,6 +55,7 @@ describe("ClickHouse migrations", () => { expect(migration_0016_error_events_4xx_and_frame_redaction.requiredForIngest).toBe(false) expect(migration_0017_error_service_version_columns.requiredForIngest).toBe(false) expect(migration_0018_apple_crash_frames.requiredForIngest).toBe(false) + expect(migration_0019_mv_sweep.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 82835fa4c..90df3b8bf 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -17,6 +17,7 @@ import { migration_0015_service_overview_minutely } from "./0015_service_overvie import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" +import { migration_0019_mv_sweep } from "./0019_mv_sweep" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -66,6 +67,7 @@ export const migrations: ReadonlyArray = [ migration_0016_error_events_4xx_and_frame_redaction, migration_0017_error_service_version_columns, migration_0018_apple_crash_frames, + migration_0019_mv_sweep, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 8600820d3..2ce4fce3a 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" as const +export const projectRevision = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -10,7 +10,6 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime),\n ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL Minute + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS error_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS logs_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS metric_catalog (\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour)\nTTL Hour + INTERVAL 90 DAY", @@ -37,7 +36,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replays (\n OrgId LowCardinality(String),\n SessionId String,\n StartTime DateTime64(9),\n EndTime Nullable(DateTime64(9)),\n DurationMs Nullable(UInt32),\n Status LowCardinality(String),\n UserId String,\n UrlInitial String,\n UserAgent String,\n BrowserName LowCardinality(String),\n OsName LowCardinality(String),\n DeviceType LowCardinality(String),\n Country LowCardinality(String) DEFAULT '',\n ServiceName LowCardinality(String),\n PageViews UInt32 DEFAULT 0,\n ClickCount UInt32 DEFAULT 0,\n ErrorCount UInt32 DEFAULT 0,\n TraceIds Array(String) DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String),\n Version UInt32,\n VisitorId String DEFAULT '',\n VisitorIsNew UInt8 DEFAULT 0,\n UserEmail String DEFAULT '',\n UserName String DEFAULT '',\n GroupId String DEFAULT '',\n GroupName String DEFAULT '',\n UserTraits Map(String, String) DEFAULT map(),\n Referrer String DEFAULT '',\n ReferrerHost LowCardinality(String) DEFAULT '',\n UtmSource LowCardinality(String) DEFAULT '',\n UtmMedium LowCardinality(String) DEFAULT '',\n UtmCampaign LowCardinality(String) DEFAULT '',\n UtmTerm String DEFAULT '',\n UtmContent String DEFAULT '',\n Host LowCardinality(String) DEFAULT '',\n EntryPath String DEFAULT '',\n ExitPath String DEFAULT '',\n Language LowCardinality(String) DEFAULT '',\n LastActivityAt Nullable(DateTime64(9))\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(StartTime)\nORDER BY (OrgId, SessionId)\nTTL toDate(StartTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", @@ -45,12 +44,11 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", @@ -73,13 +71,13 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour", - "CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix", - "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces", + "CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n -- 'traces.span.metrics.calls' is the name the collector actually emits:\n -- spanmetricsconnector output is namespaced by the pipeline it is attached\n -- to. Without it this MV matched nothing and the target sat at 0 rows since\n -- it was created, while ~880k rows / 2 days of the real counter flowed past\n -- into metrics_sum and every read fell back to the raw window-function scan\n -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with\n -- SPAN_METRICS_CALLS_NAMES on the read side.\n WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes\n FROM traces", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS\nSELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS\nSELECT\n OrgId,\n Timestamp,\n SessionId,\n Seq,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", ] as const diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index ca6d51e44..dfe68ca08 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182" as const +export const projectRevision = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" as const export const datasources = [ { @@ -34,11 +34,6 @@ export const datasources = [ content: 'DESCRIPTION >\n Minute-grain per-fingerprint error aggregates for the scheduled issue evaluator. Cascaded from error_events to avoid re-running fingerprint extraction.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime),\n ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String))\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(Minute)"\nENGINE_SORTING_KEY "OrgId, Minute, FingerprintHash"\nENGINE_TTL "Minute + INTERVAL 90 DAY"', }, - { - name: "error_spans", - content: - 'DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode=\'Error\' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, { name: "logs", content: @@ -172,7 +167,7 @@ export const datasources = [ { name: "trace_detail_spans", content: - 'DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', + 'DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', }, { name: "trace_list_mv", @@ -200,23 +195,18 @@ export const pipes = [ { name: "error_events_by_time_mv", content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", + "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", }, { name: "error_events_mv", content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events", + "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events", }, { name: "error_fingerprints_minutely_mv", content: "DESCRIPTION >\n Pre-aggregates error_events by org, minute, and fingerprint for the scheduled issue evaluator.\n\nNODE error_fingerprints_minutely_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash\n\nTYPE MATERIALIZED\nDATASOURCE error_fingerprints_minutely", }, - { - name: "error_spans_mv", - content: - "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans", - }, { name: "log_attribute_keys_mv", content: @@ -225,7 +215,7 @@ export const pipes = [ { name: "log_attribute_values_mv", content: - "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", + "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", }, { name: "logs_aggregates_hourly_mv", @@ -240,7 +230,7 @@ export const pipes = [ { name: "metric_attribute_values_mv", content: - "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", + "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", }, { name: "metric_catalog_exp_histogram_mv", @@ -355,12 +345,12 @@ export const pipes = [ { name: "span_metrics_calls_hourly_mv", content: - "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly", + "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n -- 'traces.span.metrics.calls' is the name the collector actually emits:\n -- spanmetricsconnector output is namespaced by the pipeline it is attached\n -- to. Without it this MV matched nothing and the target sat at 0 rows since\n -- it was created, while ~880k rows / 2 days of the real counter flowed past\n -- into metrics_sum and every read fell back to the raw window-function scan\n -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with\n -- SPAN_METRICS_CALLS_NAMES on the read side.\n WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly", }, { name: "trace_detail_spans_mv", content: - "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans", + "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans", }, { name: "trace_list_mv_mv", @@ -375,7 +365,7 @@ export const pipes = [ { name: "trace_resource_attribute_values_mv", content: - "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", + "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", }, { name: "trace_span_attribute_keys_mv", @@ -385,7 +375,7 @@ export const pipes = [ { name: "trace_span_attribute_values_mv", content: - "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", + "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", }, { name: "traces_aggregates_hourly_mv", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 867d6dbd6..bdc9ecdc5 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -845,37 +845,6 @@ export const serviceOverviewMinutely = defineDatasource("service_overview_minute export type ServiceOverviewMinutelyRow = InferRow -/** - * Pre-materialized error spans for the errors page. - * Pre-filters to StatusCode='Error' and pre-extracts deployment.environment - * so error queries avoid scanning the full traces table and Map columns. - * Sorted by (OrgId, ServiceName, Timestamp) for efficient filtering and aggregation. - * Populated by materialized view, not direct ingestion. - */ -export const errorSpans = defineDatasource("error_spans", { - description: - "Pre-materialized error spans for the errors page. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment. Populated by materialized view.", - jsonPaths: false, - schema: { - OrgId: t.string().lowCardinality(), - Timestamp: t.dateTime(), - TraceId: t.string(), - SpanId: t.string(), - ParentSpanId: t.string().default("__unset__"), - ServiceName: t.string().lowCardinality(), - StatusMessage: t.string(), - Duration: t.uint64(), - DeploymentEnv: t.string().lowCardinality(), - }, - engine: engine.mergeTree({ - partitionKey: "toDate(Timestamp)", - sortingKey: ["OrgId", "ServiceName", "Timestamp"], - ttl: "Timestamp + INTERVAL 90 DAY", - }), -}) - -export type ErrorSpansRow = InferRow - /** * Pre-materialized error events for the errors-as-issues triage system. * Populated from traces where StatusCode='Error'. Unwraps the first OTel @@ -1084,9 +1053,6 @@ export const traceDetailSpans = defineDatasource("trace_detail_spans", { StatusMessage: t.string(), SpanAttributes: t.map(t.string().lowCardinality(), t.string()), ResourceAttributes: t.map(t.string().lowCardinality(), t.string()), - EventsTimestamp: t.array(t.dateTime64(9)), - EventsName: t.array(t.string().lowCardinality()), - EventsAttributes: t.array(t.map(t.string().lowCardinality(), t.string())), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index cfbbfe6ba..31cdd617c 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -22,7 +22,6 @@ import { serviceOverviewSpans, serviceOverviewHourly, serviceOverviewMinutely, - errorSpans, errorEvents, errorEventsByTime, errorFingerprintsMinutely, @@ -681,36 +680,6 @@ export const servicePlatformsHourlyMv = defineMaterializedView("service_platform ], }) -/** - * Materialized view populating error_spans from error spans. - * Pre-filters to StatusCode='Error' and pre-extracts deployment.environment - * so error queries avoid scanning the full traces table and Map columns. - */ -export const errorSpansMv = defineMaterializedView("error_spans_mv", { - description: - "Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.", - datasource: errorSpans, - nodes: [ - node({ - name: "error_spans_mv_node", - sql: ` - SELECT - OrgId, - toDateTime(Timestamp) AS Timestamp, - TraceId, - SpanId, - ParentSpanId, - ServiceName, - StatusMessage, - Duration, - ResourceAttributes['deployment.environment'] AS DeploymentEnv - FROM traces - WHERE StatusCode = 'Error' - `, - }), - ], -}) - /** * Materialized view populating error_events from traces where StatusCode='Error'. * Unwraps the first OTel `exception` event and computes a cityHash64 @@ -983,10 +952,7 @@ export const traceDetailSpansMv = defineMaterializedView("trace_detail_spans_mv" StatusCode, StatusMessage, SpanAttributes, - ResourceAttributes, - EventsTimestamp, - EventsName, - EventsAttributes + ResourceAttributes FROM traces `, }), @@ -1178,7 +1144,14 @@ export const spanMetricsCallsHourlyMv = defineMaterializedView("span_metrics_cal StartTimeUnix, argMaxState(Value, TimeUnix) AS LastValue FROM metrics_sum - WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix `, }), @@ -1263,6 +1236,46 @@ export const metricCatalogExpHistogramMv = defineMaterializedView("metric_catalo ], }) +/** + * Cardinality bound shared by all four `attribute_values_hourly` materializations. + * + * That table is an AUTOCOMPLETE INDEX — it exists so the filter builder can + * suggest values for a key. It is read a couple of hundred times a week. With + * `AttributeValue != ''` as its only filter it had grown to 1.59 billion rows / + * 12.3 GB, because `ARRAY JOIN` over an attribute map turns every distinct value + * into its own row per (org, key, hour). + * + * The three rules below were chosen against what was actually in the table, not + * from intuition — measured over a 6h slice: + * + * - NUMERIC MEASUREMENTS dominated: `idle_ns` (4.9M rows) and `busy_ns` (1.7M) + * alone were ~70% of it, with `http.request.body.size` and the + * `maple.ingest.*_bytes` counters behind them. Nobody picks + * `idle_ns = 486123904` from a dropdown. Digits-only values longer than four + * characters are dropped; the threshold deliberately spares HTTP status + * codes and ports, which are low-cardinality and genuinely pickable. + * - LONG VALUES: `db.query.text` averaged 864 characters, `body` 334, + * `http.response.header.report-to` 241, `url.full` 146. Useless as + * suggestions and the bulk of the bytes. + * - UNBOUNDED IDENTIFIERS: ids and captured HTTP headers (`cf-ray`, + * `x-request-id`, `traceparent`, `date`) are unique per request by + * definition. Matched by shape rather than by an exact key list so this + * generalizes past whichever keys one customer happens to emit. + * + * This narrows VALUE suggestions only. `attribute_keys_hourly` is untouched, so + * every key stays discoverable and filterable — you just do not get a dropdown + * of values for a nanosecond counter. + * + * Regexes use `[.]` rather than an escaped dot to keep the emitted SQL free of + * backslash escaping across the TS template → DDL → chDB path. + */ +const attributeValueCardinalityBound = `WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%'` + export const logAttributeValuesMv = defineMaterializedView("log_attribute_values_mv", { description: "Aggregates log attribute values from logs hourly.", datasource: attributeValuesHourly, @@ -1281,7 +1294,7 @@ export const logAttributeValuesMv = defineMaterializedView("log_attribute_values ARRAY JOIN mapKeys(LogAttributes) AS AttributeKey, mapValues(LogAttributes) AS AttributeValue - WHERE AttributeValue != '' + ${attributeValueCardinalityBound} GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope `, }), @@ -1306,7 +1319,7 @@ export const metricAttributeValuesMv = defineMaterializedView("metric_attribute_ ARRAY JOIN mapKeys(Attributes) AS AttributeKey, mapValues(Attributes) AS AttributeValue - WHERE AttributeValue != '' + ${attributeValueCardinalityBound} GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope `, }), @@ -1331,7 +1344,7 @@ export const traceSpanAttributeValuesMv = defineMaterializedView("trace_span_att ARRAY JOIN mapKeys(SpanAttributes) AS AttributeKey, mapValues(SpanAttributes) AS AttributeValue - WHERE AttributeValue != '' + ${attributeValueCardinalityBound} GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope `, }), @@ -1356,7 +1369,7 @@ export const traceResourceAttributeValuesMv = defineMaterializedView("trace_reso ARRAY JOIN mapKeys(ResourceAttributes) AS AttributeKey, mapValues(ResourceAttributes) AS AttributeValue - WHERE AttributeValue != '' + ${attributeValueCardinalityBound} GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope `, }), diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index 4bec7be5e..dd42329d7 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -8,7 +8,6 @@ const RETENTION_DAYS = { error_events: 90, error_events_by_time: 90, error_fingerprints_minutely: 90, - error_spans: 90, logs: 30, logs_aggregates_hourly: 365, metric_catalog: 90, diff --git a/packages/otel-collector-maple-exporter/README.md b/packages/otel-collector-maple-exporter/README.md index afddb5333..06baa8fa5 100644 --- a/packages/otel-collector-maple-exporter/README.md +++ b/packages/otel-collector-maple-exporter/README.md @@ -95,7 +95,7 @@ receivers / processors plus this exporter. | OTLP signal | Maple base table | Materialized views fan-out into | | ----------- | --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Traces | `traces` | `error_events`, `error_spans`, `service_overview_spans`, `service_map_*`, `trace_list_mv`, `trace_detail_spans`, `traces_aggregates_hourly`, `service_usage`, attribute facets | +| Traces | `traces` | `error_events`, `service_overview_spans`, `service_map_*`, `trace_list_mv`, `trace_detail_spans`, `traces_aggregates_hourly`, `service_usage`, attribute facets | | Logs | `logs` | `logs_aggregates_hourly`, `service_usage`, log attribute facets | | Metrics | `metrics_sum` / `metrics_gauge` / `metrics_histogram` / `metrics_exponential_histogram` | `service_usage`, metric attribute facets | diff --git a/packages/otel-collector-maple-exporter/exporter_traces.go b/packages/otel-collector-maple-exporter/exporter_traces.go index ee331ea08..584f0548d 100644 --- a/packages/otel-collector-maple-exporter/exporter_traces.go +++ b/packages/otel-collector-maple-exporter/exporter_traces.go @@ -41,7 +41,7 @@ func (t *tracesExporter) shutdown(_ context.Context) error { return nil } // pushTraces walks the pdata.Traces tree and INSERTs one JSON row per span // into Maple's `traces` table. Materialized views inside ClickHouse fan // these out into: -// - error_events / error_spans (StatusCode = 'Error') +// - error_events / error_events_by_time (StatusCode = 'Error') // - service_overview_spans (entry-point spans) // - service_map_spans / _children (parent/child span linkage) // - service_map_edges_hourly (Client + peer.service) diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index f02f270bb..bdd283e9f 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -389,7 +389,19 @@ describe("tracesTimeseriesQuery", () => { expect(sql).not.toContain("FROM service_overview_spans") }) - it("keeps fine-grained trace timeseries on the existing MV path", () => { + // This is the shape `computeAlertBuckets` produces: one metric, sub-hour + // bucket, root spans only, and no `allMetrics`. It used to fall all the way + // through to a flat scan of the per-span `service_overview_spans` because + // `canUseAnnualServiceOverview` demanded `allMetrics === true` — 165k scans + // per 3 days against a 325M-row table while the 2.2M-row minutely rollup + // built for it sat unreachable. + // + // Asserting the minutely tier specifically, NOT just the presence of a table + // name: the tiered union reads `service_overview_spans` too (as its partial- + // bucket edge), so `toContain("FROM service_overview_spans")` passes on both + // routes and cannot tell them apart. That weak assertion is what let the + // regression sit here unnoticed. + it("routes fine-grained single-metric trace timeseries to the minutely rollup", () => { const q = tracesTimeseriesQuery({ metric: "p95_duration", needsSampling: false, @@ -397,10 +409,30 @@ describe("tracesTimeseriesQuery", () => { bucketSeconds: 300, }) const { sql } = compileCH(q, { ...baseParams, bucketSeconds: 300 }) + expect(sql).toContain("FROM service_overview_minutely") + // The raw edge covers only the partial minutes at the window's ends. expect(sql).toContain("FROM service_overview_spans") + // Sub-hour buckets have no position inside an hour-floored row. + expect(sql).not.toContain("FROM service_overview_hourly") expect(sql).not.toContain("FROM traces_aggregates_hourly") }) + // The other side of that route: a groupBy the rollup tiers cannot serve + // (`status_code` is pre-aggregated into `ErrorCount`) must still fall through + // to the flat per-span scan. + it("keeps status_code breakdowns on the flat service_overview_spans scan", () => { + const q = tracesTimeseriesQuery({ + metric: "count", + needsSampling: false, + rootOnly: true, + groupBy: ["status_code"], + bucketSeconds: 300, + }) + const { sql } = compileCH(q, { ...baseParams, bucketSeconds: 300 }) + expect(sql).toContain("FROM service_overview_spans") + expect(sql).not.toContain("FROM service_overview_minutely") + }) + it("keeps all-metrics and Apdex timeseries off traces_aggregates_hourly", () => { const allMetrics = compileCH( tracesTimeseriesQuery({ diff --git a/packages/query-engine/src/ch/queries/count-semantics.test.ts b/packages/query-engine/src/ch/queries/count-semantics.test.ts index 5ce014634..ea06d2375 100644 --- a/packages/query-engine/src/ch/queries/count-semantics.test.ts +++ b/packages/query-engine/src/ch/queries/count-semantics.test.ts @@ -35,6 +35,13 @@ function countExpr(sql: string): string { return line.slice(0, line.indexOf(" AS count")).trim() } +/** The SELECT expression aliased as `spanCount`, e.g. `count()`. */ +function spanCountExpr(sql: string): string { + const line = sql.split("\n").find((l) => / AS spanCount\b/.test(l)) + if (!line) throw new Error(`no spanCount column in:\n${sql}`) + return line.slice(0, line.indexOf(" AS spanCount")).trim() +} + function sourceTable(sql: string): string { const tables = [...sql.matchAll(/FROM (\w+)/g)].map((m) => m[1]) // Union/CTE shapes read several tables; the invariant cares about the set. @@ -64,8 +71,26 @@ const TIMESERIES_ROUTES: ReadonlyArray<{ opts: { metric: "count", needsSampling: false, groupBy: ["http_method"], bucketSeconds: 3600 }, }, { + // `status_code` is what keeps this on the flat per-span scan: the overview + // rollup tiers pre-aggregate it away into `ErrorCount`, so they cannot group + // by it. Without a groupBy outside `OVERVIEW_ROLLUP_GROUP_KEYS` these opts + // now route to the tiers below. name: "service_overview_spans MV", table: "service_overview_spans", + opts: { + metric: "count", + needsSampling: false, + rootOnly: true, + groupBy: ["status_code"], + bucketSeconds: 300, + }, + }, + { + // The alert-evaluation shape: one metric, sub-hour bucket, root spans only. + // `computeAlertBuckets` never sets `allMetrics`, which used to force this + // onto the per-span scan above; it now reads the minutely rollup instead. + name: "annual service overview union (single metric, no allMetrics)", + table: "service_overview_minutely+service_overview_spans", opts: { metric: "count", needsSampling: false, rootOnly: true, bucketSeconds: 300 }, }, { @@ -115,13 +140,28 @@ describe("traces count is sample-weighted on every route", () => { // `spanCount` (rows observed), never the extrapolated `count`. Row-level // tables must therefore keep both columns distinct. it("keeps an unweighted spanCount alongside the weighted count", () => { + // Asserted as an invariant rather than a literal, because the routes differ + // in how they spell it: row-level tables emit `count()`, while the overview + // tiers sum a stored raw `SpanCount` through `bCount`. Both are unweighted, + // which is the property `minimumSampleCount` depends on. for (const opts of [ { metric: "count", needsSampling: false, groupBy: ["http_method"], bucketSeconds: 300 }, { metric: "count", needsSampling: false, rootOnly: true, bucketSeconds: 300 }, + { + metric: "count", + needsSampling: false, + rootOnly: true, + groupBy: ["status_code"], + bucketSeconds: 300, + }, ] as const) { const { sql } = compileCH(tracesTimeseriesQuery(opts), { ...baseParams, bucketSeconds: 300 }) - expect(sql).toContain("count() AS spanCount") - expect(countExpr(sql)).toBe("sum(SampleRate)") + const spanCount = spanCountExpr(sql) + expect( + /SampleRate|Estimated|Weighted/.test(spanCount), + `spanCount expression "${spanCount}" is sample-weighted; it is the confidence guard and must count observed rows`, + ).toBe(false) + expectWeighted(sql) } }) diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 9360d8bc7..6ab5dfdf8 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -767,7 +767,7 @@ export function errorsFacetsQuery(opts: ErrorsFacetsOpts): CHUnionQuery = new Set(["service", "non * Reading the hourly tier for a sub-hour bucket would pile every interior hour * onto the bucket containing `:00` and leave the rest of the hour reading zero. * + * No longer gated on `allMetrics === true`. That flag only picks which aggregates + * the *raw* paths bother computing (`metricSelectExprs`); the tiers here read + * pre-aggregated columns and produce all five `MetricNeed`s unconditionally, so + * every `TracesMetric` is serveable. Requiring it kept alert evaluation + * (`computeAlertBuckets`, which never sets it) off this route entirely and on a + * flat scan of the per-span `service_overview_spans` — 165k scans / 3 days. + * The apdex threshold check below is the real capability limit: `rawEdges` + * hardcodes the 500ms buckets the rollups store. + * + * Unlike the `traces_aggregates_hourly` route, all three tiers here carry a true + * raw `SpanCount` alongside the weighted one, so a rule routed here evaluates + * `minimumSampleCount` against a real sample count rather than an estimate. + * * Exported because it names a distinct SQL *route*: the SQL it selects is * structurally unlike every other branch of `tracesTimeseriesQuery`, so the * catalog sweep in `sql-catalog.ts` asserts a fixture exercises it both ways. @@ -393,8 +406,16 @@ export function canUseAnnualServiceOverview(opts: TracesTimeseriesOpts): boolean const tierIsAvailable = opts.overviewTiers === "hour" ? bucketSeconds % 3600 === 0 : bucketSeconds % 60 === 0 + // A single-metric caller on an hour-multiple bucket is better served by + // `traces_aggregates_hourly` (the next branch): it stores sample-WEIGHTED + // quantile states, where these tiers store unweighted ones. Yield those to it + // and claim only the sub-hour case, where the alternative is a per-span scan + // of `service_overview_spans`. `allMetrics` callers stay here at every bucket + // because that route cannot serve them at all. + const prefersAggregatesHourly = opts.allMetrics !== true && bucketSeconds % 3600 === 0 + return ( - opts.allMetrics === true && + !prefersAggregatesHourly && (opts.apdexThresholdMs ?? 500) === 500 && opts.rootOnly === true && bucketSeconds >= 60 && diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 25f7f0696..0b60a046a 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -54,9 +54,6 @@ export const TraceDetailSpans = table("trace_detail_spans", { StatusMessage: T.string, SpanAttributes: T.map(T.string, T.string), ResourceAttributes: T.map(T.string, T.string), - EventsTimestamp: T.array(T.dateTime64), - EventsName: T.array(T.string), - EventsAttributes: T.array(T.map(T.string, T.string)), }) export const TraceListMv = table("trace_list_mv", { @@ -154,18 +151,6 @@ export const ServiceOverviewMinutely = table("service_overview_minutely", { ApdexToleratingCount: T.uint64, }) -export const ErrorSpans = table("error_spans", { - OrgId: T.string, - Timestamp: T.dateTime, - TraceId: T.string, - SpanId: T.string, - ParentSpanId: T.string, - ServiceName: T.string, - StatusMessage: T.string, - Duration: T.uint64, - DeploymentEnv: T.string, -}) - export const ErrorEvents = table("error_events", { OrgId: T.string, Timestamp: T.dateTime, diff --git a/packages/query-engine/src/datetime.ts b/packages/query-engine/src/datetime.ts index 195bb6df7..fbc6335bf 100644 --- a/packages/query-engine/src/datetime.ts +++ b/packages/query-engine/src/datetime.ts @@ -601,6 +601,29 @@ export function formatBucketSecondsShort(seconds: number): string { */ export const alertWindowBucketSeconds = (windowMinutes: number): number => Math.max(windowMinutes * 60, 60) +/** The scheduler tick period. Alert windows are snapped to it — see below. */ +export const ALERT_TICK_SECONDS = 60 + +/** + * Snap an alert evaluation window's end to the tick boundary. + * + * Two reasons, one of which is a correctness improvement rather than a cache + * trick: + * + * 1. FRESHNESS — a window ending at `now` always includes the current, + * still-filling minute, so the final bucket reads low for reasons that have + * nothing to do with the signal. The error tick already excludes it + * (`ErrorsService`'s cutoff floors to the minute and subtracts one more). + * 2. SHARING — every rule in a tick computed its own `now`, so rules over the + * identical query still produced distinct windows and distinct cache keys. + * Snapping makes a tick's rules agree, which is what lets the `qe-evaluate` + * entry be reused by, say, warn-at-100 and page-at-500 on one signal. + * + * Costs up to one tick of extra lag, which is bounded by the tick period the + * scheduler already imposes. + */ +export const snapAlertWindowEndMs = (epochMs: number): number => floorToBucketMs(epochMs, ALERT_TICK_SECONDS) + const floorToBucketMs = (epochMs: number, bucketSeconds: number): number => { const bucketMs = bucketSeconds * 1000 return Math.floor(epochMs / bucketMs) * bucketMs