diff --git a/.agents/skills/maple-telemetry-conventions/SKILL.md b/.agents/skills/maple-telemetry-conventions/SKILL.md index 0443fd909..1cea2bd30 100644 --- a/.agents/skills/maple-telemetry-conventions/SKILL.md +++ b/.agents/skills/maple-telemetry-conventions/SKILL.md @@ -21,7 +21,7 @@ Reference for the **language-agnostic** OpenTelemetry conventions Maple uses acr - `rules/span-attributes.md` — Master reference of every custom attribute key Maple emits, grouped by namespace, with file:line citations. - `rules/status-and-kind.md` — Title Case status code rule (`Ok`/`Error`/`Unset`) and span kind conventions (`Server` / `Client` / `Internal`). -- `rules/resource-attributes.md` — `service.*` identity, `deployment.environment.name` resolution order, the legacy `deployment.environment` dual-emit, and `maple_org_id`. +- `rules/resource-attributes.md` — `service.*` identity, `deployment.environment.name` resolution order, the deprecated `deployment.environment` dual-emit and read-side coalesce, and `maple_org_id`. - `rules/language-bindings.md` — Parallel TypeScript / Rust / Python snippets that emit the same attribute keys. - `rules/mv-first-class-columns.md` — Which span and resource attributes Tinybird MVs pre-extract into columns (and the rule for adding new ones). - `rules/service-map-attribution.md` — Required span and resource attributes for the service map to render edges, runtime icons, and platform badges. Includes the canonical `peer.service` registry. @@ -35,7 +35,7 @@ Reference for the **language-agnostic** OpenTelemetry conventions Maple uses acr | Vendor namespace | Custom attributes go under `maple.*`. Sub-namespaces: `maple.ingest.*`, `maple.cloudflare.*`. | | Standard semconv | Use OTel semconv keys verbatim: `service.name`, `http.request.method`, `db.system.name`, `error.type`. | | Org identity | `orgId` (camelCase) in TypeScript spans, `maple.org_id` (dotted) in Rust spans. Don't unify until MVs migrate. | -| Deployment env | Dual-emit `deployment.environment` + `deployment.environment.name`. Keep both until MV `coalesce()` migration lands. | +| Deployment env | Emit `deployment.environment.name` (we dual-emit the deprecated `deployment.environment` too). **Read** both via `DEPLOYMENT_ENV_SQL` / `deploymentEnvExpr` — never a bare map lookup. | | Warehouse SQL spans | Every span from `WarehouseQueryService.executeSql` carries `db.system.name`, `peer.service`, `db.query.text`, `db.query.fingerprint`, `db.duration_ms`, `result.rowCount`, `orgId`, `query.context`, `query.profile`. Legacy spans (pre 2026-06) use `db.statement*`/`db.system`; warehouse readers coalesce both. | | Service map | Outbound spans need `peer.service` (HTTP/RPC) or `db.system.name` (DB) on a `Client`/`Producer` span. Resource attrs need `process.runtime.name`, `cloud.platform`, `maple.sdk.type` for runtime icon + platform badge. See `rules/service-map-attribution.md`. | | Loop prevention | Never remove `HttpMiddleware.TracerDisabledWhen` (apps/api/src/app.ts:169-175) or the ingest loopback guard (apps/ingest/src/main.rs:499-514). | diff --git a/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md b/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md index 4c0e39ed7..2d2ccd438 100644 --- a/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md +++ b/.agents/skills/maple-telemetry-conventions/rules/mv-first-class-columns.md @@ -15,7 +15,7 @@ Lightweight projection of trace spans for the service dependency map. | Column | Extracted from | Line | |---|---|---| | `PeerService` | `SpanAttributes['peer.service']` | `materializations.ts:246` | -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:247` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:247` | ## `service_map_children_mv` @@ -23,7 +23,7 @@ Spans with a parent, for child-of-edge analysis in the service map. | Column | Extracted from | Line | |---|---|---| -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:315` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:315` | ## `service_map_edges_hourly_mv` @@ -32,7 +32,7 @@ Pre-aggregated client-to-peer edges. Pre-filters to spans with `peer.service != | Column | Extracted from | Line | |---|---|---| | `TargetService` | `SpanAttributes['peer.service']` | `materializations.ts:341` | -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:342` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:342` | ## `service_overview_spans_mv` @@ -40,7 +40,7 @@ Hourly service-overview rollup. | Column | Extracted from | Line | |---|---|---| -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:277` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:277` | | `CommitSha` | `ResourceAttributes['deployment.commit_sha']` | `materializations.ts:278` (see file for the field) | ## `service_platforms_hourly_mv` @@ -49,7 +49,7 @@ Per-service hosting-platform attributes for the service map's runtime-icon resol | Column | Extracted from | Line | |---|---|---| -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:414` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:414` | | `K8sCluster` | `max(ResourceAttributes['k8s.cluster.name'])` | `materializations.ts:415` | | `K8sPodName` | `max(ResourceAttributes['k8s.pod.name'])` | `materializations.ts:416` | | `K8sDeploymentName` | `max(ResourceAttributes['k8s.deployment.name'])` | `materializations.ts:417` | @@ -69,7 +69,7 @@ Unwraps the first OTel `exception` event from `EventsName` / `EventsAttributes` | `ExceptionStacktrace` | first `exception` event's `exception.stacktrace` | `materializations.ts:499` | | `TopFrame` | (computed from stacktrace) | `materializations.ts` | | `FingerprintHash` | cityHash64 of grouping keys | `materializations.ts:489` | -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:525` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:525` | ## `trace_list_mv` @@ -80,7 +80,7 @@ Trace list optimized for the trace search UI. Pre-filters to entry-point spans. | `HttpMethod` | `SpanAttributes['http.method']` (fallback `http.request.method`) | `materializations.ts:596` | | `HttpRoute` | `SpanAttributes['http.route']` (fallbacks `url.path`, `http.target`) | `materializations.ts:597` | | `HttpStatusCode` | `SpanAttributes['http.status_code']` (fallback `http.response.status_code`) | `materializations.ts:598` | -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:599` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:599` | ## `traces_aggregates_hourly_mv` @@ -88,13 +88,13 @@ Hourly trace-shape rollup. | Column | Extracted from | Line | |---|---|---| -| `DeploymentEnv` | `ResourceAttributes['deployment.environment']` | `materializations.ts:778` | +| `DeploymentEnv` | `DEPLOYMENT_ENV_SQL` | `materializations.ts:778` | Plus dimension keys pre-aggregated on the trace itself: `ServiceName`, `SpanName`, `SpanKind`, `StatusCode`, `IsEntryPoint`. ## `logs_aggregates_hourly_mv` (referenced) -Same pattern — extracts `DeploymentEnv` from `ResourceAttributes['deployment.environment']`. See `materializations.ts:809`. +Same pattern — extracts `DeploymentEnv` through `DEPLOYMENT_ENV_SQL`. See `materializations.ts:809`. --- @@ -104,7 +104,7 @@ Same pattern — extracts `DeploymentEnv` from `ResourceAttributes['deployment.e Example: if you add a new MV column `HttpUserAgent` that extracts `SpanAttributes['http.user_agent']`, every service emitting spans **must** use `http.user_agent` as the attribute key. Don't have one service emit `http.user_agent` and another `userAgent` — only one will populate the column. -Corollary: **the dual-emit rule for `deployment.environment` is here**. Every MV in this file extracts `ResourceAttributes['deployment.environment']` (the legacy key). Until those MVs migrate to `coalesce(deployment.environment.name, deployment.environment)`, the legacy resource attribute **must** be emitted alongside the new one. See `rules/resource-attributes.md`. +Corollary: every MV here extracts `DeploymentEnv` through the shared `DEPLOYMENT_ENV_SQL` fragment (`packages/domain/src/tinybird/semconv-renames.ts`), which coalesces `deployment.environment.name` over the deprecated `deployment.environment` — as `MESSAGING_DESTINATION_SQL` does for `messaging.destination(.name)` in the external-edge rollup. A bare lookup on either key alone silently materializes an empty environment for half the instrumentation in the wild. See `rules/resource-attributes.md`. ## When NOT to extract into a column diff --git a/.agents/skills/maple-telemetry-conventions/rules/resource-attributes.md b/.agents/skills/maple-telemetry-conventions/rules/resource-attributes.md index 74e137619..83763c862 100644 --- a/.agents/skills/maple-telemetry-conventions/rules/resource-attributes.md +++ b/.agents/skills/maple-telemetry-conventions/rules/resource-attributes.md @@ -11,14 +11,27 @@ Resource attributes are set **once per process** on the OTel `Resource` and appl | `service.version` | string | build-time | `env!("CARGO_PKG_VERSION")` in Rust, package version in TS | Semantic version; used for release-correlation. | | `service.instance.id` | string | runtime | `uuid::Uuid::new_v4().to_string()` per process | Per-process UUID generated at startup. Lets dashboards distinguish replicas. | -## Deployment environment — dual emit ⚠ +## Deployment environment — read both, emit both -The most important rule in this file. Maple emits **both** the new and legacy keys: +OTel renamed this attribute; the registry lists `deployment.environment.name` as stable and plain `deployment.environment` as deprecated ("Replaced by `deployment.environment.name`"). | Key | Status | |---|---| -| `deployment.environment.name` | OTel-canonical (new spec) | -| `deployment.environment` | Legacy — **keep until all Tinybird MVs migrate to `coalesce()` both keys** | +| `deployment.environment.name` | OTel-canonical — emit this one from new code | +| `deployment.environment` | Deprecated — Maple still emits it, and still **reads** it | + +**Reading is the rule that matters.** Anything that pulls the environment out of `ResourceAttributes` — an MV body, a query-engine filter, a facet — uses the shared coalesce, never a bare map lookup: + +```ts +import { DEPLOYMENT_ENV_SQL, deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" +``` + +```sql +coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), + ResourceAttributes['deployment.environment']) +``` + +A bare lookup on either key alone drops the environment for half the fleet: our own SDKs dual-emit, a current OTel SDK sends only `.name`, an older one only the legacy key. `packages/domain/src/tinybird/semconv-renames.test.ts` fails if any MV extracts `DeploymentEnv` some other way. The same module carries `MESSAGING_DESTINATION_SQL` for the `messaging.destination` -> `messaging.destination.name` rename; any future rename Maple *keys on* belongs there too. Source: `apps/ingest/src/main.rs:526-538` @@ -27,12 +40,10 @@ Source: `apps/ingest/src/main.rs:526-538` "deployment.environment.name", deployment_env.clone(), )) -// Dual-emit the legacy `deployment.environment` key: every Tinybird MV -// (service_overview_spans_mv, service_map_*_mv, error_*_mv, -// logs_aggregates_hourly_mv, service_platforms_hourly_mv) still -// pre-extracts ResourceAttributes['deployment.environment'] at write -// time, so omitting it leaves DeploymentEnv='' for every ingest span -// and the services-table badge renders blank. +// Dual-emit the deprecated `deployment.environment` key: the MVs +// coalesce both spellings since migration 0020, but rows materialized +// before it — and pre-0020 BYO-ClickHouse schemas — still read the +// legacy key alone. .with_attribute(OtelKeyValue::new("deployment.environment", deployment_env)) ``` @@ -58,16 +69,7 @@ Any new service must follow the same priority order. Don't read `NODE_ENV` or `E ### When can the dual-emit be dropped? -When every MV in `packages/domain/src/tinybird/materializations.ts` that currently extracts `ResourceAttributes['deployment.environment']` switches to: - -```sql -coalesce( - ResourceAttributes['deployment.environment.name'], - ResourceAttributes['deployment.environment'] -) AS DeploymentEnv -``` - -…and the migration backfills any historical data. Until then, **never remove the legacy emit** — it leaves `DeploymentEnv=''` in the services table and the deployment-env badge renders blank. +ClickHouse migration 0020 moved every MV onto the coalesce, so our own rollups no longer depend on the legacy emit. Two things still do: rows those MVs materialized **before** 0020 (they carry whatever the legacy key held and age out with the target TTL), and BYO-ClickHouse orgs whose schema is still pre-0020. Dropping the legacy emit is a follow-up gated on both, not a free cleanup. ## `maple_org_id` — internal service identity diff --git a/CLAUDE.md b/CLAUDE.md index a8e33d76d..ccc654082 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,8 +162,11 @@ to per-request hot paths like token validation. OTLP export bypasses the API, so `apps/ingest` (Rust) self-instruments over OTLP/HTTP to its own `INGEST_FORWARD_OTLP_ENDPOINT` (startup guard refuses a loopback endpoint), as `service.name="ingest"`, `maple_org_id="internal"`, -and `deployment.environment.name` **dual-emitted** as the legacy `deployment.environment` because -the Tinybird MVs still pre-extract the old key. Custom fields use the `maple.*` namespace. Span +and `deployment.environment.name` **dual-emitted** as the deprecated `deployment.environment` +(the MVs coalesce both since migration 0020 — see `DEPLOYMENT_ENV_SQL` in +`packages/domain/src/tinybird/semconv-renames.ts`, which is where every +renamed-key coalesce belongs; the dual-emit remains for rows already +materialized under the old key and for pre-0020 BYO-ClickHouse schemas). Custom fields use the `maple.*` namespace. Span status follows OTEL HTTP semconv for SERVER spans: **only 5xx is `Error`, 4xx rejections are `Ok`** (`otel_status_for_rejection` in `apps/ingest/src/main.rs`) so error dashboards aren't flooded by expected 401/402/429s. Operational metrics (`apps/ingest/src/metrics.rs`) push via OTLP every 30s — diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 7db633f51..77458cd5f 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -86,4 +86,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "f1bdef1ca3dcb073fc3cda998c145f64fd9a4a4f174543e7bc0405867bedf356", projectRevision: "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979", }), + Object.freeze({ + version: 10, + fingerprint: "b10c1137fb63a6f3", + digest: "b10c1137fb63a6f3ebd4d44c37ed82d469d1e52ea483c03b3c5cc25399410bd1", + manifestDigest: "7c975d2547ccdedb55f5b5fe2360fa72a4e6b9b7fd1935611e257e207c5694fd", + projectRevision: "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 76baaad5f..0dc2359d1 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 9 as const +export const LOCAL_SCHEMA_VERSION = 10 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index e46100d36..c616fc206 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -45,6 +45,7 @@ import { v5ToV6ErrorEventsFingerprintHygieneModule } from "./local-store-migrati import { v6ToV7ErrorServiceVersionModule } from "./local-store-migrations/v6-to-v7-error-service-version" import { v7ToV8AppleCrashFramesModule } from "./local-store-migrations/v7-to-v8-apple-crash-frames" import { v8ToV9MvSweepModule } from "./local-store-migrations/v8-to-v9-mv-sweep" +import { v9ToV10SemconvKeyRenamesModule } from "./local-store-migrations/v9-to-v10-semconv-key-renames" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -111,6 +112,7 @@ export const localStoreMigrations: ReadonlyArray = v6ToV7ErrorServiceVersionModule, v7ToV8AppleCrashFramesModule, v8ToV9MvSweepModule, + v9ToV10SemconvKeyRenamesModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts b/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts new file mode 100644 index 000000000..793f46574 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts @@ -0,0 +1,205 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { + LOCAL_SCHEMA_V9, + LOCAL_SCHEMA_V9_MANIFEST, + LOCAL_SCHEMA_V9_SQL, + LOCAL_SCHEMA_V10, + LOCAL_SCHEMA_V10_MANIFEST, + LOCAL_SCHEMA_V10_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0009-to-0010-semconv-key-renames" as const + +const V9ToV10StateCodec = makeRawRowsState(MODULE_ID) + +type V9ToV10State = typeof V9ToV10StateCodec.schema.Type +type V9ToV10Progress = InstalledProgress + +const decodeState = V9ToV10StateCodec.decode +const decodeProgress = decodeInstalledProgress + +/** + * Every view whose body reads one of the renamed keys. + * + * The bundled v10 DDL is `CREATE ... IF NOT EXISTS` throughout, so a view whose + * body changed has to be dropped first or the v9 version simply survives — the + * same trap the v8 -> v9 edge documents. No table or column changes here: the + * `TO` targets are identical in v9 and v10. + * + * chDB materializes views as tables, so `DROP TABLE` is the right verb. + */ +const DROPPED_VIEWS = [ + "error_events_by_time_mv", + "error_events_mv", + "logs_aggregates_hourly_mv", + "service_external_edges_hourly_mv", + "service_map_children_mv", + "service_map_db_edges_hourly_mv", + "service_map_db_query_shapes_hourly_mv", + "service_map_spans_mv", + "service_operations_minutely_mv", + "service_overview_hourly_mv", + "service_overview_minutely_mv", + "service_overview_spans_mv", + "service_platforms_hourly_mv", + "trace_list_mv_mv", + "traces_aggregates_hourly_mv", +] as const + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V9_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V9_SQL, bootstrapSchema: false }, + ) + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V9ToV10State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * The local mirror of ClickHouse migration 0020: every view that pre-extracts + * `DeploymentEnv` now reads `deployment.environment.name` with the deprecated + * `deployment.environment` as fallback, and the external-edge rollup reads + * `messaging.destination.name` with `messaging.destination` as fallback — + * instead of the deprecated spelling alone in both cases. + * + * Forward-only. Rows already materialized keep what the v9 bodies wrote — an + * empty environment for a service that only ever sent the canonical key, a + * system-labelled edge for a current producer span — and the targets converge as + * their TTL rolls. That is the same position a deployed cluster is in right + * after migration 0020, and rebuilding them would mean rewriting a store this + * edge has just promised to clone byte-for-byte. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + for (const view of DROPPED_VIEWS) db.exec(`DROP TABLE IF EXISTS ${view}`) + }, + { schemaSql: LOCAL_SCHEMA_V9_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V10_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V9ToV10State, + _progress: V9ToV10Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V10_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v9 -> v10 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V10_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v9-store", + description: "Clone the stopped v9 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "rebuild-semconv-rename-views", + description: "Rebuild every view that reads a renamed OTel key so it accepts both spellings", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v10-schema", + description: "Verify the v10 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v9 store is cloned byte-for-byte before any view is replaced.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of every replaced view is neither read nor rewritten; only view definitions change.", + }, + { + name: "service and trace rollups", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows are preserved untouched; buckets materialized after the migration resolve both the environment and the messaging destination under either semconv spelling, and rows written with an empty DeploymentEnv or a system-labelled target age out with the rollup TTL.", + preservationInterval: "rollup retention horizon", + sourceRetentionDays: 365, + targetRetentionDays: 365, + }, +] + +export const v9ToV10SemconvKeyRenamesModule: LocalStoreMigrationModule = { + id: MODULE_ID, + moduleVersion: 1, + description: "Rebuild every view that reads a renamed OTel attribute to accept both spellings", + from: LOCAL_SCHEMA_V9, + to: LOCAL_SCHEMA_V10, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index cd4118c2e..659087651 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -8,6 +8,7 @@ import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import schemaV7Sql from "./schema/local-schema-v7.sql" with { type: "text" } import schemaV8Sql from "./schema/local-schema-v8.sql" with { type: "text" } import schemaV9Sql from "./schema/local-schema-v9.sql" with { type: "text" } +import schemaV10Sql from "./schema/local-schema-v10.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -31,7 +32,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" + "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -46,7 +47,7 @@ export const LOCAL_SCHEMA_MANIFEST_DIGEST = LOCAL_SCHEMA_MANIFEST.digest * Immutable per-version DDL and manifest snapshots. * * A historical edge must keep constructing and verifying the schema it was - * written for: when v9 ships, v7 -> v8 must still produce v8 rather than + * written for: when v10 ships, v8 -> v9 must still produce v9 rather than * silently retargeting whatever the generator currently emits. The SQL is * imported literally because Bun resolves text imports statically; everything * derived from it is built once, here. @@ -61,6 +62,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV7Sql, schemaV8Sql, schemaV9Sql, + schemaV10Sql, ] export interface LocalSchemaSnapshot { @@ -112,6 +114,9 @@ export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = snapshotAt(8).manifestDigest export const LOCAL_SCHEMA_V9_SQL = snapshotAt(9).sql export const LOCAL_SCHEMA_V9_MANIFEST = snapshotAt(9).manifest export const LOCAL_SCHEMA_V9_MANIFEST_DIGEST = snapshotAt(9).manifestDigest +export const LOCAL_SCHEMA_V10_SQL = snapshotAt(10).sql +export const LOCAL_SCHEMA_V10_MANIFEST = snapshotAt(10).manifest +export const LOCAL_SCHEMA_V10_MANIFEST_DIGEST = snapshotAt(10).manifestDigest export interface LocalSchemaIdentity { readonly version: number @@ -125,7 +130,7 @@ export interface LocalSchemaIdentity { /** * Per-version identities, frozen and read straight from the append-only * history. Historical migration edges must never point at - * CURRENT_LOCAL_SCHEMA: when v9 ships, v0 -> v1 must still construct and verify + * CURRENT_LOCAL_SCHEMA: when v10 ships, v0 -> v1 must still construct and verify * v1 rather than silently changing its destination. */ const identityAt = (version: number): LocalSchemaIdentity => { @@ -151,6 +156,7 @@ export const LOCAL_SCHEMA_V6 = identityAt(6) export const LOCAL_SCHEMA_V7 = identityAt(7) export const LOCAL_SCHEMA_V8 = identityAt(8) export const LOCAL_SCHEMA_V9 = identityAt(9) +export const LOCAL_SCHEMA_V10 = identityAt(10) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index ba5d601cc..115d4d245 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979", + "projectRevision": "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v10.sql b/apps/cli/src/server/schema/local-schema-v10.sql new file mode 100644 index 000000000..d5f7371cf --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v10.sql @@ -0,0 +1,1833 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3 +-- localSchemaVersion: 10 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 26a2f3bbc..d5f7371cf 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979 --- localSchemaVersion: 9 +-- projectRevision: 5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3 +-- localSchemaVersion: 10 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -951,7 +951,7 @@ WITH SpanId, ParentSpanId, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, _exType AS ExceptionType, _exMsg AS ExceptionMessage, _exStack AS ExceptionStacktrace, @@ -1091,7 +1091,7 @@ WITH SpanId, ParentSpanId, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, _exType AS ExceptionType, _exMsg AS ExceptionMessage, _exStack AS ExceptionStacktrace, @@ -1170,7 +1170,7 @@ SELECT toStartOfHour(TimestampTime) AS Hour, ServiceName, SeverityText, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, count() AS Count, sum(length(Body) + 200) AS SizeBytes, ResourceAttributes['service.namespace'] AS ServiceNamespace @@ -1278,18 +1278,18 @@ SELECT toStartOfHour(toDateTime(Timestamp)) AS Hour, ServiceName, multiIf( - SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', 'http' ) AS TargetType, multiIf( - SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], '' ) AS TargetSystem, multiIf( - SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', - if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), if(SpanAttributes['server.address'] != '', @@ -1298,7 +1298,7 @@ SELECT SpanAttributes['http.host'], SpanAttributes['url.authority'])) ) AS TargetName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, count() AS CallCount, countIf(StatusCode = 'Error') AS ErrorCount, sum(Duration / 1000000) AS DurationSumMs, @@ -1312,7 +1312,7 @@ SELECT SpanAttributes['server.address'] != '' OR SpanAttributes['http.host'] != '' OR SpanAttributes['url.authority'] != '' - OR SpanAttributes['messaging.destination'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '' OR SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '' @@ -1331,7 +1331,7 @@ SELECT Duration, StatusCode, TraceState, - ResourceAttributes['deployment.environment'] AS DeploymentEnv + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv FROM traces WHERE SpanKind IN ('Server', 'Consumer') AND ParentSpanId != ''; @@ -1343,7 +1343,7 @@ SELECT ServiceName, coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, count() AS CallCount, countIf(StatusCode = 'Error') AS ErrorCount, sum(Duration / 1000000) AS DurationSumMs, @@ -1364,7 +1364,7 @@ SELECT ServiceName, coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, coalesce( nullIf(SpanAttributes['db.query.fingerprint'], ''), nullIf(SpanAttributes['db.statement.fingerprint'], ''), @@ -1429,7 +1429,7 @@ SELECT Duration, StatusCode, TraceState, - ResourceAttributes['deployment.environment'] AS DeploymentEnv + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv FROM traces WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); @@ -1454,7 +1454,7 @@ SELECT OrgId, toStartOfMinute(toDateTime(Timestamp)) AS Minute, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, count() AS SpanCount, sum(SampleRate) AS EstimatedSpanCount, @@ -1470,7 +1470,7 @@ SELECT OrgId, toStartOfHour(toDateTime(Timestamp)) AS Hour, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, ResourceAttributes['service.namespace'] AS ServiceNamespace, ResourceAttributes['deployment.commit_sha'] AS CommitSha, count() AS SpanCount, @@ -1491,7 +1491,7 @@ SELECT OrgId, toStartOfMinute(toDateTime(Timestamp)) AS Minute, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, ResourceAttributes['service.namespace'] AS ServiceNamespace, ResourceAttributes['deployment.commit_sha'] AS CommitSha, count() AS SpanCount, @@ -1515,7 +1515,7 @@ SELECT Duration, StatusCode, TraceState, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, ResourceAttributes['deployment.commit_sha'] AS CommitSha, SampleRate, ResourceAttributes['service.namespace'] AS ServiceNamespace @@ -1527,7 +1527,7 @@ SELECT OrgId, toStartOfHour(toDateTime(Timestamp)) AS Hour, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, @@ -1725,7 +1725,7 @@ SELECT if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, toUInt8( StatusCode = 'Error' OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) @@ -1807,7 +1807,7 @@ SELECT SpanKind, StatusCode, IsEntryPoint, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, sum(SampleRate) AS WeightedCount, sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 53a662554..e7dc1a129 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -20,7 +20,7 @@ import { LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V7, LOCAL_SCHEMA_V8, - LOCAL_SCHEMA_V9, + LOCAL_SCHEMA_V10, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -62,16 +62,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v9 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("2516215f22b41a63") - expect(SCHEMA_DIGEST).toBe("2516215f22b41a636b3186d0b293a0a6276e4bb85004efd3994b80867696a469") + it("matches the generated v10 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("b10c1137fb63a6f3") + expect(SCHEMA_DIGEST).toBe("b10c1137fb63a6f3ebd4d44c37ed82d469d1e52ea483c03b3c5cc25399410bd1") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(9) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V9) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(10) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V10) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -188,6 +188,7 @@ describe("local migration registry", () => { "local-0006-to-0007-error-service-version", "local-0007-to-0008-apple-crash-frames", "local-0008-to-0009-mv-sweep", + "local-0009-to-0010-semconv-key-renames", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -234,7 +235,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 10, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 11, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 8967a6159..0afec69d6 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 9 and .schema == "2516215f22b41a63"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 10 and .schema == "b10c1137fb63a6f3"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index be0a5cf86..93e9f288d 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979"; +pub const PROJECT_REVISION: &str = "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/ingest/src/otel.rs b/apps/ingest/src/otel.rs index e1c4b55b9..d6d2784bf 100644 --- a/apps/ingest/src/otel.rs +++ b/apps/ingest/src/otel.rs @@ -21,11 +21,11 @@ pub fn build_resource(cfg: ResourceConfig) -> Resource { KeyValue::new("service.version", cfg.service_version), KeyValue::new("service.instance.id", cfg.service_instance_id), KeyValue::new("deployment.environment.name", cfg.deployment_env.clone()), - // Dual-emit the legacy `deployment.environment` key — every Tinybird MV - // (service_overview_spans_mv, service_map_*_mv, error_*_mv, - // logs_aggregates_hourly_mv, service_platforms_hourly_mv) still pre-extracts - // ResourceAttributes['deployment.environment'] at write time. Drop only - // after those MVs migrate to coalesce() both keys. + // Dual-emit the deprecated `deployment.environment` key. Since ClickHouse + // migration 0020 the MVs coalesce both spellings, so this is no longer + // load-bearing for OUR rollups — it stays for the rows those MVs already + // materialized under the legacy key, and for customer queries and BYO + // ClickHouse deployments still pinned to a pre-0020 schema. KeyValue::new("deployment.environment", cfg.deployment_env), KeyValue::new("process.runtime.name", "rust"), KeyValue::new("process.runtime.version", rustc_version()), diff --git a/apps/landing/src/content/docs/sdks/effect-client.md b/apps/landing/src/content/docs/sdks/effect-client.md index c92b746d2..c903b8642 100644 --- a/apps/landing/src/content/docs/sdks/effect-client.md +++ b/apps/landing/src/content/docs/sdks/effect-client.md @@ -37,7 +37,7 @@ The `/client` import is required — the bare `@maple-dev/effect-sdk` import res The client layer reads from `globalThis.navigator` and `Intl.DateTimeFormat` to populate resource attributes automatically: -- `browser.user_agent` — `navigator.userAgent` +- `user_agent.original` — `navigator.userAgent` - `browser.language` — `navigator.language` - `browser.timezone` — `Intl.DateTimeFormat().resolvedOptions().timeZone` - `maple.sdk.type` — always `"client"`, so server- and browser-emitted spans can be filtered apart diff --git a/docs/attributes.md b/docs/attributes.md index 8b0663645..617a03dda 100644 --- a/docs/attributes.md +++ b/docs/attributes.md @@ -23,11 +23,11 @@ Tag every span with these and you get per-environment and per-version slices for | Attribute | Example | What Maple does with it | | ----------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------- | -| `deployment.environment` | `production` | Filterable everywhere; per-env throughput / latency / error rate; environment chips in span detail. | -| `deployment.environment.name` | `production` | Accepted as an alias for `deployment.environment` in infrastructure queries. | +| `deployment.environment.name` | `production` | The current OTel key. Filterable everywhere; per-env throughput / latency / error rate; environment chips in span detail. | +| `deployment.environment` | `production` | The deprecated spelling of the same attribute. Read everywhere the `.name` key is, so older instrumentation keeps working. | | `deployment.commit_sha` | `c0b92f68` | Per-version metrics in service overview; exposed as the `commit_sha` discovery facet so you can pivot by deploy. | -Maple's SDKs currently dual-emit both `deployment.environment` and `deployment.environment.name` — they're treated as the same value. Emitting either is fine. In the search bar, `env`, `environment`, and `commit_sha` are short aliases — see [Filter aliases](#filter-aliases) below. +OpenTelemetry renamed this attribute (`deployment.environment` → `deployment.environment.name`); Maple reads whichever one your spans carry, preferring `.name` when both are present, so emitting either is fine. Maple's own SDKs dual-emit both. The same holds for `messaging.destination` → `messaging.destination.name` on messaging spans. In the search bar, `env`, `environment`, and `commit_sha` are short aliases — see [Filter aliases](#filter-aliases) below. ## Span status & kind diff --git a/docs/otel-spec/emerging-and-compat.md b/docs/otel-spec/emerging-and-compat.md index 1a4d43dd0..0cfae1cf6 100644 --- a/docs/otel-spec/emerging-and-compat.md +++ b/docs/otel-spec/emerging-and-compat.md @@ -26,7 +26,7 @@ hand. - **Telemetry schemas are the principled version of what we do by hand.** CLAUDE.md documents that `WarehouseQueryService.executeSql` spans "coalesce both" legacy `db.statement*`/`db.system` and current `db.query.text`/`db.system.name` spellings, and that semconv renamed - `deployment.environment` → `deployment.environment.name` (dual-emitted by `apps/ingest`). That + `deployment.environment` → `deployment.environment.name` (dual-emitted by `apps/ingest`, coalesced on read by `DEPLOYMENT_ENV_SQL`). That is exactly the `rename_attributes` transform a schema file encodes formally. We do not currently read `schema_url` off ingested `ResourceSpans`/`ResourceLogs`/`ResourceMetrics` to drive this automatically — it's manual per-migration logic. This doc is the reference for what a diff --git a/docs/otel-spec/resource-and-config.md b/docs/otel-spec/resource-and-config.md index 3106e903d..d17d3972f 100644 --- a/docs/otel-spec/resource-and-config.md +++ b/docs/otel-spec/resource-and-config.md @@ -5,7 +5,7 @@ This page is a compliance reference for the OpenTelemetry **Resource** model (wh ## Relevance to Maple - Maple is primarily a **consumer/backend** of OTel data (Rust `ingest` gateway → collector → ClickHouse/Tinybird → web dashboard), plus we **self-instrument** our own services (`apps/ingest`, the API, etc.) — so we need to both parse resources correctly from arbitrary SDKs _and_ emit spec-correct resources ourselves. -- **Dashboards key off `service.name` + `deployment.environment.name`.** The legacy `deployment.environment` attribute is deprecated (replaced by `deployment.environment.name`); our ingest gateway dual-emits both today because downstream Tinybird materialized views (`service_overview_spans_mv` et al.) still pre-extract the legacy key — see `CLAUDE.md`. This doc's deprecation citation below is the spec basis for eventually dropping the legacy emission once those MVs coalesce both. +- **Dashboards key off `service.name` + `deployment.environment.name`.** The legacy `deployment.environment` attribute is deprecated (replaced by `deployment.environment.name`); our ingest gateway dual-emits both today, and every materialized view coalesces the two (`DEPLOYMENT_ENV_SQL` in `packages/domain/src/tinybird/semconv-renames.ts`, ClickHouse migration 0020) so telemetry carrying either key resolves to the same environment — see `CLAUDE.md`. This doc's deprecation citation below is the spec basis for eventually dropping the legacy emission, which now waits only on pre-0020 rows aging out and on BYO-ClickHouse schemas catching up. - Our ingest gateway stamps a **vendor-namespaced** `maple_org_id` (and `maple.*` span attributes) — this is intentionally outside semconv (no `maple.*` resource semantic convention exists upstream), so it must never collide with a real OTel resource attribute name. - We do not currently consume declarative/file configuration or the Entities data model, but both are tracked here because they affect how upstream SDKs will emit resources in the near future (Entities) and how our own self-instrumentation could be configured (file config). @@ -57,7 +57,7 @@ Source: https://opentelemetry.io/docs/specs/semconv/resource/deployment-environm - **`deployment.environment.name`** — **Stable**, Recommended, type `string`. "Name of the deployment environment (aka deployment tier)." Well-known values (all Stable): `development`, `production`, `staging`, `test` — a well-known value MUST be used when it applies; otherwise a custom value MAY be used. - Explicit note: this attribute "does not affect the uniqueness constraints defined through the `service.namespace`, `service.name` and `service.instance.id` attributes" — i.e. the same logical service running in two environments is still considered the same service identity for uniqueness purposes; the environment is an orthogonal dimension. -- **Legacy `deployment.environment`** (no `.name`) is registered as **Deprecated**, with the exact registry note: **"Replaced by `deployment.environment.name`."** This is the citable spec basis for the rename Maple's ingest gateway currently works around by dual-emitting both keys (see `CLAUDE.md` self-observability section) — compliant new code should emit only `deployment.environment.name`. +- **Legacy `deployment.environment`** (no `.name`) is registered as **Deprecated**, with the exact registry note: **"Replaced by `deployment.environment.name`."** This is the citable spec basis for the rename Maple handles by dual-emitting both keys on the write side and coalescing them on every read (see `CLAUDE.md` self-observability section) — compliant new code should emit only `deployment.environment.name`. - The broader "Deployment" resource/entity group itself is at **Development** status overall (per the resource semconv landing page), even though the two attributes above are individually Stable — Development status here reflects unfinished entity-role modeling, not attribute-level instability. ### 3.3 `telemetry.sdk.*` / `telemetry.distro.*` diff --git a/packages/domain/package.json b/packages/domain/package.json index 2f1c408dc..826da412a 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -23,6 +23,7 @@ "./tinybird": "./src/tinybird/index.ts", "./tinybird/db-query-shape-sql": "./src/tinybird/db-query-shape-sql.ts", "./tinybird/fingerprint": "./src/tinybird/fingerprint.ts", + "./tinybird/semconv-renames": "./src/tinybird/semconv-renames.ts", "./tinybird/span-display-name": "./src/tinybird/span-display-name.ts", "./clickhouse": "./src/clickhouse/index.ts", "./generated/clickhouse-schema": "./src/generated/clickhouse-schema.ts", diff --git a/packages/domain/src/clickhouse/migrations/0020_semconv_key_renames.ts b/packages/domain/src/clickhouse/migrations/0020_semconv_key_renames.ts new file mode 100644 index 000000000..976d33bb8 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0020_semconv_key_renames.ts @@ -0,0 +1,87 @@ +import type { ClickHouseMigration } from "./index" + +/** + * Migration 0020 — read renamed OTel attributes under either spelling. + * + * Semantic conventions rename keys; instrumentation in the wild does not update + * in lockstep. Two renames were being read on the deprecated spelling alone, so + * telemetry from a current OTel SDK lost the attribute at write time — inside a + * materialized view, where the loss is invisible and permanent for those rows. + * + * 1. **`deployment.environment` -> `deployment.environment.name`.** The registry + * lists the `.name` key as stable and the bare one as deprecated ("Replaced + * by `deployment.environment.name`"). Every MV that pre-extracts + * `DeploymentEnv` pinned the deprecated key, so a service instrumented with an + * SDK new enough to have adopted the rename materialized an EMPTY environment + * into every rollup — no environment facet, no environment filter, on exactly + * the dashboards the rollups back. Maple's own SDKs and the ingest gateway + * dual-emit, which is why this never showed up on our own telemetry. + * + * 2. **`messaging.destination` -> `messaging.destination.name`.** The messaging + * semconv namespaced the destination. `service_external_edges_hourly_mv` reads + * it for `TargetName`, so a current producer span fell through to the + * *system* value and every topic or queue collapsed into one `kafka` / `sqs` + * node instead of a per-destination edge. Maple's own `VcsSyncQueue` spans emit + * the `.name` key, so this one was visible on our own service map. + * + * Both now read through the shared fragments in + * `packages/domain/src/tinybird/semconv-renames.ts`, e.g. + * + * coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), + * ResourceAttributes['deployment.environment']) + * + * canonical key first, deprecated fallback — byte-identical to the read side's + * `deploymentEnvExpr` / `messagingDestinationExpr`, so a raw-table branch and an + * MV branch of the same union still produce the same grouping key. + * + * An MV body can only be changed by dropping and recreating the view, so each + * pair here is `DROP VIEW` + `CREATE MATERIALIZED VIEW`. The `TO` targets are + * untouched: no data is dropped, and inserts arriving inside the drop/create gap + * simply do not materialize for that view (the same exposure migration 0019 had). + * + * Forward-only. Rows already materialized keep what the old bodies wrote; they + * age out with the target's TTL. Backfilling would mean rewriting every rollup + * from `traces`, which costs far more than waiting out a 90d/365d TTL for a + * column that was empty or mislabeled on those rows anyway. + * + * `requiredForIngest: false` — every table here is MV-populated and the raw + * insert path is unchanged, so a server mid-migration keeps ingesting. + */ +export const migration_0020_semconv_key_renames: ClickHouseMigration = { + version: 20, + description: + "Read deployment.environment(.name) and messaging.destination(.name) under either semconv spelling", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_events_by_time_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "DROP VIEW IF EXISTS error_events_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "DROP VIEW IF EXISTS logs_aggregates_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace", + "DROP VIEW IF EXISTS service_external_edges_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", + "DROP VIEW IF EXISTS service_map_children_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", + "DROP VIEW IF EXISTS service_map_db_edges_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", + "DROP VIEW IF EXISTS service_map_db_query_shapes_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n)))\n) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey", + "DROP VIEW IF EXISTS service_map_spans_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')", + "DROP VIEW IF EXISTS service_operations_minutely_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName", + "DROP VIEW IF EXISTS service_overview_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", + "DROP VIEW IF EXISTS service_overview_minutely_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", + "DROP VIEW IF EXISTS service_overview_spans_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", + "DROP VIEW IF EXISTS service_platforms_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", + "DROP VIEW IF EXISTS trace_list_mv_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS\nSELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''", + "DROP VIEW IF EXISTS traces_aggregates_hourly_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 3447c2b15..70dd83a41 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -25,6 +25,7 @@ import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_erro import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" import { migration_0019_mv_sweep } from "./0019_mv_sweep" +import { migration_0020_semconv_key_renames } from "./0020_semconv_key_renames" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -40,12 +41,12 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ]) - expect(migrations.at(-1)).toBe(migration_0019_mv_sweep) - expect(latestMigrationVersion).toBe(19) - // 0010 and 0014-0019 are read-path only, so the ingest-gating version skips - // all seven and stays at 13 — nothing writes `web_events`, + expect(migrations.at(-1)).toBe(migration_0020_semconv_key_renames) + expect(latestMigrationVersion).toBe(20) + // 0010 and 0014-0020 are read-path only, so the ingest-gating version skips + // all eight and stays at 13 — nothing writes `web_events`, // `service_overview_minutely` or `error_events` directly, and bumping it // would un-ready every BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") @@ -56,6 +57,7 @@ describe("ClickHouse migrations", () => { expect(migration_0017_error_service_version_columns.requiredForIngest).toBe(false) expect(migration_0018_apple_crash_frames.requiredForIngest).toBe(false) expect(migration_0019_mv_sweep.requiredForIngest).toBe(false) + expect(migration_0020_semconv_key_renames.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 90df3b8bf..59b84ddf9 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -18,6 +18,7 @@ import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_erro import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" import { migration_0019_mv_sweep } from "./0019_mv_sweep" +import { migration_0020_semconv_key_renames } from "./0020_semconv_key_renames" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -68,6 +69,7 @@ export const migrations: ReadonlyArray = [ migration_0017_error_service_version_columns, migration_0018_apple_crash_frames, migration_0019_mv_sweep, + migration_0020_semconv_key_renames, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 2ce4fce3a..aff0fa313 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" as const +export const projectRevision = "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -41,30 +41,30 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace", + "CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n)))\n) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n)))\n) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS\nSELECT\n OrgId,\n Hour,\n SourceService,\n TargetService,\n DeploymentEnv,\n CallCount,\n ErrorCount,\n DurationSumMs,\n MaxDurationMs,\n SampledSpanCount,\n UnsampledSpanCount,\n SampleRateSum\n FROM service_map_edges_hourly_ingest", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS\nSELECT\n OrgId,\n toStartOfHour(Minute) AS Hour,\n ServiceName,\n DeploymentEnv,\n SpanName,\n sum(SpanCount) AS SpanCount,\n sum(EstimatedSpanCount) AS EstimatedSpanCount,\n sum(ErrorCount) AS ErrorCount,\n sum(EstimatedErrorCount) AS EstimatedErrorCount,\n sum(DurationSum) AS DurationSum,\n quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles\n FROM service_operations_minutely\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", @@ -73,11 +73,11 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n -- 'traces.span.metrics.calls' is the name the collector actually emits:\n -- spanmetricsconnector output is namespaced by the pipeline it is attached\n -- to. Without it this MV matched nothing and the target sat at 0 rows since\n -- it was created, while ~880k rows / 2 days of the real counter flowed past\n -- into metrics_sum and every read fell back to the raw window-function scan\n -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with\n -- SPAN_METRICS_CALLS_NAMES on the read side.\n WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes\n FROM traces", - "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS\nSELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS\nSELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", "CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n AND length(AttributeValue) <= 128\n AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$'))\n AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$')\n AND AttributeKey NOT LIKE 'http.request.header.%'\n AND AttributeKey NOT LIKE 'http.response.header.%'\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope", - "CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS\nSELECT\n OrgId,\n Timestamp,\n SessionId,\n Seq,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", ] as const diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index dfe68ca08..5294dad7c 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "3773cd0bfa79483773ada07c70c9fa5571688ceecfb1fd5839c33c4251b7f979" as const +export const projectRevision = "5b4c3a0d3aa0962b062689605ad5cf075f47403df04e851ce58133f16fc692e3" as const export const datasources = [ { @@ -195,12 +195,12 @@ export const pipes = [ { name: "error_events_by_time_mv", content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", + "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", }, { name: "error_events_mv", content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events", + "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events", }, { name: "error_fingerprints_minutely_mv", @@ -220,7 +220,7 @@ export const pipes = [ { name: "logs_aggregates_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly", + "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly", }, { name: "metric_attribute_keys_mv", @@ -255,22 +255,22 @@ export const pipes = [ { name: "service_external_edges_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly", + "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly", }, { name: "service_map_children_mv", content: - "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children", + "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children", }, { name: "service_map_db_edges_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly", + "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly", }, { name: "service_map_db_query_shapes_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly", + "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly", }, { name: "service_map_edges_hourly_ingest_mv", @@ -280,7 +280,7 @@ export const pipes = [ { name: "service_map_spans_mv", content: - "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans", + "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans", }, { name: "service_operations_hourly_mv", @@ -290,27 +290,27 @@ export const pipes = [ { name: "service_operations_minutely_mv", content: - "DESCRIPTION >\n Pre-aggregates every span by service operation and minute with normalized HTTP names, exact/estimated counts, errors, duration sum, and unweighted t-digest state.\n\nNODE service_operations_minutely_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName\n\nTYPE MATERIALIZED\nDATASOURCE service_operations_minutely", + "DESCRIPTION >\n Pre-aggregates every span by service operation and minute with normalized HTTP names, exact/estimated counts, errors, duration sum, and unweighted t-digest state.\n\nNODE service_operations_minutely_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName\n\nTYPE MATERIALIZED\nDATASOURCE service_operations_minutely", }, { name: "service_overview_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates service entry-point spans hourly by environment, namespace, and commit for one-year service history.\n\nNODE service_overview_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_hourly", + "DESCRIPTION >\n Pre-aggregates service entry-point spans hourly by environment, namespace, and commit for one-year service history.\n\nNODE service_overview_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_hourly", }, { name: "service_overview_minutely_mv", content: - "DESCRIPTION >\n Pre-aggregates service entry-point spans minutely by environment, namespace, and commit, for windows whose bucket size is under an hour.\n\nNODE service_overview_minutely_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_minutely", + "DESCRIPTION >\n Pre-aggregates service entry-point spans minutely by environment, namespace, and commit, for windows whose bucket size is under an hour.\n\nNODE service_overview_minutely_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_minutely", }, { name: "service_overview_spans_mv", content: - "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans", + "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans", }, { name: "service_platforms_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", + "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", }, { name: "service_usage_logs_mv", @@ -355,7 +355,7 @@ export const pipes = [ { name: "trace_list_mv_mv", content: - "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv", + "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv", }, { name: "trace_resource_attribute_keys_mv", @@ -380,7 +380,7 @@ export const pipes = [ { name: "traces_aggregates_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly", + "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly", }, { name: "web_events_mv", diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index 369d78efd..baed8edf8 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -126,8 +126,9 @@ export const MetricsFilters = Schema.Struct({ metricType: MetricType, serviceName: Schema.optional(ServiceName), // Metrics tables have no pre-extracted DeploymentEnv column, so this lowers to - // a predicate on `ResourceAttributes['deployment.environment']` (see - // `metricsTimeseriesQuery`). Same field name as TracesFilters/LogsFilters. + // a predicate on `DEPLOYMENT_ENV_SQL` over `ResourceAttributes` — either + // semconv spelling of the key (see `metricsTimeseriesQuery`). Same field name + // as TracesFilters/LogsFilters. environments: Schema.optional(Schema.Array(DeploymentEnvironment)), groupByAttributeKey: Schema.optional(Schema.String), // Resource-attribute counterpart of `groupByAttributeKey` — groups by a diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index bdc9ecdc5..1d5bff60b 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -580,7 +580,7 @@ export type ServiceMapDbQuerySignaturesHourlyRow = InferRow= 500) @@ -1403,7 +1404,7 @@ export const tracesAggregatesHourlyMv = defineMaterializedView("traces_aggregate SpanKind, StatusCode, IsEntryPoint, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ${DEPLOYMENT_ENV_SQL} AS DeploymentEnv, sum(SampleRate) AS WeightedCount, sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, @@ -1434,7 +1435,7 @@ export const serviceOperationsMinutelyMv = defineMaterializedView("service_opera OrgId, toStartOfMinute(toDateTime(Timestamp)) AS Minute, ServiceName, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ${DEPLOYMENT_ENV_SQL} AS DeploymentEnv, ${NORMALIZED_SPAN_NAME_SQL} AS SpanName, count() AS SpanCount, sum(SampleRate) AS EstimatedSpanCount, @@ -1497,7 +1498,7 @@ export const logsAggregatesHourlyMv = defineMaterializedView("logs_aggregates_ho toStartOfHour(TimestampTime) AS Hour, ServiceName, SeverityText, - ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ${DEPLOYMENT_ENV_SQL} AS DeploymentEnv, count() AS Count, sum(length(Body) + 200) AS SizeBytes, ResourceAttributes['service.namespace'] AS ServiceNamespace diff --git a/packages/domain/src/tinybird/semconv-renames.test.ts b/packages/domain/src/tinybird/semconv-renames.test.ts new file mode 100644 index 000000000..19b39af7e --- /dev/null +++ b/packages/domain/src/tinybird/semconv-renames.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/clickhouse-builder/expr" +import { compile } from "@maple-dev/clickhouse-builder/sql" +import { + DEPLOYMENT_ENV_SQL, + MESSAGING_DESTINATION_SQL, + deploymentEnvExpr, + messagingDestinationExpr, +} from "./semconv-renames" +import { latestSnapshotStatements } from "../generated/clickhouse-schema" + +describe("semconv rename coalescing", () => { + it("prefers the stable key and falls back to the deprecated one", () => { + expect(DEPLOYMENT_ENV_SQL).toBe( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment'])", + ) + }) + + it("prefers the namespaced messaging destination over the deprecated one", () => { + expect(MESSAGING_DESTINATION_SQL).toBe( + "coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination'])", + ) + }) + + // Byte-identity is load-bearing, not cosmetic: the service-map and + // service-overview routes union an MV branch with a raw-`traces` branch, and + // the two only merge if both sides compute the same grouping key. + it("compiles byte-identically from the DSL expressions the read side uses", () => { + const mapColumn = (name: string) => { + const column = CH.dynamicColumn>(name) + return { get: (key: string) => CH.mapGet(column, key) } + } + expect(compile(deploymentEnvExpr(mapColumn("ResourceAttributes")).toFragment())).toBe( + DEPLOYMENT_ENV_SQL, + ) + expect(compile(messagingDestinationExpr(mapColumn("SpanAttributes")).toFragment())).toBe( + MESSAGING_DESTINATION_SQL, + ) + }) + + // The regression this guards: an MV that reads only the deprecated key + // materializes an EMPTY environment for any service instrumented with an OTel + // SDK new enough to have adopted the rename — and the rollups are what the + // dashboards read. See ClickHouse migration 0020. + it("leaves no materialized view reading a deprecated key on its own", () => { + const bare = latestSnapshotStatements + .filter((statement) => statement.includes("CREATE MATERIALIZED VIEW")) + .filter((statement) => statement.includes("SpanAttributes['messaging.destination']")) + .filter((statement) => !statement.includes(MESSAGING_DESTINATION_SQL)) + .map((statement) => statement.match(/VIEW IF NOT EXISTS (\w+)/)?.[1]) + expect(bare).toEqual([]) + }) + + it("is the only way a materialized view extracts DeploymentEnv", () => { + const offenders = latestSnapshotStatements + .filter((statement) => statement.includes("CREATE MATERIALIZED VIEW")) + .filter((statement) => statement.includes("AS DeploymentEnv")) + .filter((statement) => !statement.includes(`${DEPLOYMENT_ENV_SQL} AS DeploymentEnv`)) + .map((statement) => statement.match(/VIEW IF NOT EXISTS (\w+)/)?.[1]) + expect(offenders).toEqual([]) + }) +}) diff --git a/packages/domain/src/tinybird/semconv-renames.ts b/packages/domain/src/tinybird/semconv-renames.ts new file mode 100644 index 000000000..66cb7afb5 --- /dev/null +++ b/packages/domain/src/tinybird/semconv-renames.ts @@ -0,0 +1,85 @@ +// Coalescing expressions for OpenTelemetry attribute renames. +// +// Semconv renames keys; instrumentation in the wild does not update in lockstep. +// Anything that reads a renamed key has to accept BOTH spellings, or it silently +// loses the attribute for whichever half of the fleet uses the other one — an +// empty environment column, a service-map node collapsed onto the wrong label. +// Each expression here is used in two places that MUST agree byte-for-byte: +// the materialized view that pre-extracts the value at write time, and the +// query-engine builder that reads it off the raw table. +// +// Renames whose keys are only ever surfaced (never keyed on) live in the +// recommendation dictionary in `../recommendations.ts` instead. + +import type { Expr } from "@maple-dev/clickhouse-builder/expr" +import * as CH from "@maple-dev/clickhouse-builder/expr" +import { compile } from "@maple-dev/clickhouse-builder/sql" + +/** + * The shape every caller shares: a query builder's `$.ResourceAttributes` / + * `$.SpanAttributes` column accessor, and the bare-column stand-in below that + * the generated SQL text is compiled from. + */ +interface MapColumnLike { + get(key: string): Expr +} + +const mapColumn = (name: string): MapColumnLike => { + const column = CH.dynamicColumn>(name) + return { get: (key) => CH.mapGet(column, key) } +} + +/** + * Canonical deployment-environment expression, shared by the materialized views + * that pre-extract `DeploymentEnv` at write time and by the runtime queries that + * read the environment straight off `ResourceAttributes`. + * + * OpenTelemetry renamed the resource attribute: `deployment.environment.name` is + * the stable key, and the registry marks plain `deployment.environment` as + * deprecated ("Replaced by `deployment.environment.name`"). Both spellings are in + * the wild — Maple's own SDKs and the ingest gateway dual-emit, an OTel SDK new + * enough to have adopted the rename sends only `.name`, and everything older + * sends only the legacy key. Reading either one alone drops an environment for + * half the fleet, which is what this expression exists to prevent. + * + * Map lookups return `''` for a missing key, so the `nullIf` is what turns + * "absent" into a `coalesce` fallback. The last argument is non-nullable, so the + * whole expression stays `String` — it can feed a non-Nullable MV column. + */ +export function deploymentEnvExpr(resourceAttributes: MapColumnLike): Expr { + return CH.coalesce( + CH.nullIf(resourceAttributes.get("deployment.environment.name"), ""), + resourceAttributes.get("deployment.environment"), + ) +} + +/** + * SQL text required by Tinybird materialization and ClickHouse migration DDL. + * Compiles byte-identically to {@link deploymentEnvExpr} applied to the raw + * `ResourceAttributes` column, so an MV's pre-extracted `DeploymentEnv` and the + * read side's raw-table fallback agree on every row. + */ +export const DEPLOYMENT_ENV_SQL = compile(deploymentEnvExpr(mapColumn("ResourceAttributes")).toFragment()) + +/** + * Canonical messaging-destination expression. + * + * The messaging semconv namespaced the destination as `messaging.destination.name` + * and deprecated the bare `messaging.destination`. The service-map external-edge + * rollup keys its `TargetName` off this: reading only the deprecated spelling + * made every topic and queue from current instrumentation fall back to the + * *system* value, collapsing per-destination edges into a single `kafka` / `sqs` + * node. Maple's own producer spans (`VcsSyncQueue`) emit the `.name` key, so the + * gap was visible on our own service map. + */ +export function messagingDestinationExpr(spanAttributes: MapColumnLike): Expr { + return CH.coalesce( + CH.nullIf(spanAttributes.get("messaging.destination.name"), ""), + spanAttributes.get("messaging.destination"), + ) +} + +/** SQL text for {@link messagingDestinationExpr} over the raw `SpanAttributes` column. */ +export const MESSAGING_DESTINATION_SQL = compile( + messagingDestinationExpr(mapColumn("SpanAttributes")).toFragment(), +) diff --git a/packages/effect-sdk/src/client/flushable.test.ts b/packages/effect-sdk/src/client/flushable.test.ts index b1c946c84..e39b622e5 100644 --- a/packages/effect-sdk/src/client/flushable.test.ts +++ b/packages/effect-sdk/src/client/flushable.test.ts @@ -228,7 +228,7 @@ describe("MapleFlush.make (client)", () => { } ).resourceSpans[0].resource.attributes const attrMap = Object.fromEntries(attrs.map((a) => [a.key, a.value.stringValue])) - expect(attrMap["browser.user_agent"]).toBe("TestAgent/1.0") + expect(attrMap["user_agent.original"]).toBe("TestAgent/1.0") expect(attrMap["browser.language"]).toBe("en-GB") // Intl is always present in node/browsers; just assert it's a non-empty string. expect(typeof attrMap["browser.timezone"]).toBe("string") diff --git a/packages/effect-sdk/src/client/flushable.ts b/packages/effect-sdk/src/client/flushable.ts index df994146e..37faf36d5 100644 --- a/packages/effect-sdk/src/client/flushable.ts +++ b/packages/effect-sdk/src/client/flushable.ts @@ -154,7 +154,9 @@ const buildBrowserAttributes = (config: MapleClientFlushableConfig): Record const nav = browserNavigator() if (nav) { - if (nav.userAgent) attributes["browser.user_agent"] = nav.userAgent + // `user_agent.original` is the semconv key; `browser.user_agent` was + // deprecated in favour of it. + if (nav.userAgent) attributes["user_agent.original"] = nav.userAgent if (nav.language) attributes["browser.language"] = nav.language } if (typeof Intl !== "undefined") { diff --git a/packages/effect-sdk/src/client/layer.ts b/packages/effect-sdk/src/client/layer.ts index c2bcd9b6f..5911c634a 100644 --- a/packages/effect-sdk/src/client/layer.ts +++ b/packages/effect-sdk/src/client/layer.ts @@ -84,7 +84,9 @@ export const layer = (config: MapleClientConfig) => { } satisfies Record const nav = browserNavigator() if (nav) { - if (nav.userAgent) attributes["browser.user_agent"] = nav.userAgent + // `user_agent.original` is the semconv key; `browser.user_agent` was + // deprecated in favour of it. + if (nav.userAgent) attributes["user_agent.original"] = nav.userAgent if (nav.language) attributes["browser.language"] = nav.language } if (typeof Intl !== "undefined") { diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 776650da1..be8b73a43 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -212,7 +212,7 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:infra:nodeFacetsQuery:default [ecfd3ec1] +-- builder:infra:nodeFacetsQuery:default [483ce011] SELECT ResourceAttributes['k8s.node.name'] AS name, uniq(ResourceAttributes['k8s.node.name']) AS count, @@ -246,7 +246,7 @@ SELECT LIMIT 50 UNION ALL SELECT - ResourceAttributes['deployment.environment.name'] AS name, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS name, uniq(ResourceAttributes['k8s.node.name']) AS count, 'environment' AS facetType FROM metrics_gauge @@ -256,7 +256,7 @@ SELECT AND ResourceAttributes['k8s.node.name'] != '' AND ResourceAttributes['k8s.pod.name'] = '' AND MetricName IN ('k8s.node.cpu.usage') - AND ResourceAttributes['deployment.environment.name'] != '' + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) != '' GROUP BY name ORDER BY count DESC LIMIT 50 @@ -278,7 +278,7 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:infra:podFacetsQuery:default [b49456ea] +-- builder:infra:podFacetsQuery:default [c7a651d6] SELECT ResourceAttributes['k8s.pod.name'] AS name, uniq(ResourceAttributes['k8s.pod.uid']) AS count, @@ -400,7 +400,7 @@ SELECT LIMIT 100 UNION ALL SELECT - ResourceAttributes['deployment.environment.name'] AS name, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS name, uniq(ResourceAttributes['k8s.pod.uid']) AS count, 'environment' AS facetType FROM metrics_gauge @@ -409,7 +409,7 @@ SELECT AND TimeUnix <= '2026-01-03 14:15:00' AND ResourceAttributes['k8s.pod.name'] != '' AND MetricName IN ('k8s.pod.cpu.usage') - AND ResourceAttributes['deployment.environment.name'] != '' + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) != '' GROUP BY name ORDER BY count DESC LIMIT 50 @@ -446,7 +446,7 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:infra:workloadFacetsQuery:default [e1f768b0] +-- builder:infra:workloadFacetsQuery:default [b424bb44] SELECT ResourceAttributes['k8s.deployment.name'] AS name, uniq(ResourceAttributes['k8s.deployment.name']) AS count, @@ -493,7 +493,7 @@ SELECT LIMIT 50 UNION ALL SELECT - ResourceAttributes['deployment.environment.name'] AS name, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS name, uniq(ResourceAttributes['k8s.deployment.name']) AS count, 'environment' AS facetType FROM metrics_gauge @@ -502,7 +502,7 @@ SELECT AND TimeUnix <= '2026-01-03 14:15:00' AND ResourceAttributes['k8s.deployment.name'] != '' AND MetricName IN ('k8s.pod.cpu.usage') - AND ResourceAttributes['deployment.environment.name'] != '' + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) != '' GROUP BY name ORDER BY count DESC LIMIT 50 @@ -915,7 +915,7 @@ SELECT LIMIT 200 FORMAT JSON --- builder:service-map:serviceExternalEdgesSQL:default [4894e4c9] +-- builder:service-map:serviceExternalEdgesSQL:default [9e00902d] SELECT sourceService AS sourceService, targetType AS targetType, @@ -947,9 +947,9 @@ SELECT UNION ALL SELECT ServiceName AS sourceService, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), 'messaging', (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), 'rpc', 'http') AS targetType, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), SpanAttributes['messaging.system'], (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), SpanAttributes['rpc.system'], '') AS targetSystem, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), if(SpanAttributes['server.address'] != '', SpanAttributes['server.address'], if(SpanAttributes['http.host'] != '', SpanAttributes['http.host'], SpanAttributes['url.authority']))) AS targetName, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), 'messaging', (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), 'rpc', 'http') AS targetType, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), SpanAttributes['messaging.system'], (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), SpanAttributes['rpc.system'], '') AS targetSystem, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), if(SpanAttributes['server.address'] != '', SpanAttributes['server.address'], if(SpanAttributes['http.host'] != '', SpanAttributes['http.host'], SpanAttributes['url.authority']))) AS targetName, count() AS bucketCallCount, countIf(StatusCode = 'Error') AS bucketErrorCount, sum(Duration / 1000000) AS bucketDurationSumMs, @@ -962,7 +962,7 @@ SELECT AND Timestamp <= '2026-01-03 14:15:00' AND SpanKind IN ('Client', 'Producer') AND SpanAttributes['db.system.name'] = '' - AND ((((((SpanAttributes['server.address'] != '' OR SpanAttributes['http.host'] != '') OR SpanAttributes['url.authority'] != '') OR SpanAttributes['messaging.destination'] != '') OR SpanAttributes['messaging.system'] != '') OR SpanAttributes['rpc.service'] != '') OR SpanAttributes['rpc.system'] != '') + AND ((((((SpanAttributes['server.address'] != '' OR SpanAttributes['http.host'] != '') OR SpanAttributes['url.authority'] != '') OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '') OR SpanAttributes['messaging.system'] != '') OR SpanAttributes['rpc.service'] != '') OR SpanAttributes['rpc.system'] != '') GROUP BY sourceService, targetType, targetSystem, targetName HAVING targetName != '' ) AS edges @@ -980,7 +980,7 @@ SELECT LIMIT 200 FORMAT JSON --- builder:service-map:serviceExternalEdgesSQL:env-scoped [2a7ffabc] +-- builder:service-map:serviceExternalEdgesSQL:env-scoped [f135f8aa] SELECT sourceService AS sourceService, targetType AS targetType, @@ -1013,9 +1013,9 @@ SELECT UNION ALL SELECT ServiceName AS sourceService, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), 'messaging', (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), 'rpc', 'http') AS targetType, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), SpanAttributes['messaging.system'], (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), SpanAttributes['rpc.system'], '') AS targetSystem, - multiIf((SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != ''), if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), if(SpanAttributes['server.address'] != '', SpanAttributes['server.address'], if(SpanAttributes['http.host'] != '', SpanAttributes['http.host'], SpanAttributes['url.authority']))) AS targetName, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), 'messaging', (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), 'rpc', 'http') AS targetType, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), SpanAttributes['messaging.system'], (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), SpanAttributes['rpc.system'], '') AS targetSystem, + multiIf((coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != ''), if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), (SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != ''), if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), if(SpanAttributes['server.address'] != '', SpanAttributes['server.address'], if(SpanAttributes['http.host'] != '', SpanAttributes['http.host'], SpanAttributes['url.authority']))) AS targetName, count() AS bucketCallCount, countIf(StatusCode = 'Error') AS bucketErrorCount, sum(Duration / 1000000) AS bucketDurationSumMs, @@ -1028,8 +1028,8 @@ SELECT AND Timestamp <= '2026-01-03 14:15:00' AND SpanKind IN ('Client', 'Producer') AND SpanAttributes['db.system.name'] = '' - AND ((((((SpanAttributes['server.address'] != '' OR SpanAttributes['http.host'] != '') OR SpanAttributes['url.authority'] != '') OR SpanAttributes['messaging.destination'] != '') OR SpanAttributes['messaging.system'] != '') OR SpanAttributes['rpc.service'] != '') OR SpanAttributes['rpc.system'] != '') - AND ResourceAttributes['deployment.environment'] = 'production' + AND ((((((SpanAttributes['server.address'] != '' OR SpanAttributes['http.host'] != '') OR SpanAttributes['url.authority'] != '') OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '') OR SpanAttributes['messaging.system'] != '') OR SpanAttributes['rpc.service'] != '') OR SpanAttributes['rpc.system'] != '') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) = 'production' GROUP BY sourceService, targetType, targetSystem, targetName HAVING targetName != '' ) AS edges @@ -1197,7 +1197,7 @@ SELECT LIMIT 50 FORMAT JSON --- builder:service-operations:serviceOperationsSummaryQuery:envFiltered [66a9f885] +-- builder:service-operations:serviceOperationsSummaryQuery:envFiltered [54970693] SELECT bSpanName AS spanName, sum(bSpanCount) AS spanCount, @@ -1222,7 +1222,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfMinute(toDateTime('2026-01-01 10:30:00')), toStartOfMinute(toDateTime('2026-01-01 10:30:00')), toStartOfMinute(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 MINUTE) OR Timestamp >= toStartOfMinute(toDateTime('2026-01-03 14:15:00'))) GROUP BY bSpanName UNION ALL @@ -4446,7 +4446,7 @@ SELECT LIMIT 100 FORMAT JSON --- pipe:list_traces:filtered:baseline [b1aaf6f5] +-- pipe:list_traces:filtered:baseline [53aa2fe9] SELECT TraceId AS traceId, Timestamp AS startTime, @@ -4472,7 +4472,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET' AND ResourceAttributes['service.namespace'] = 'core' AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -4487,7 +4487,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET' AND ResourceAttributes['service.namespace'] = 'core' ORDER BY ts DESC @@ -4496,7 +4496,7 @@ SELECT LIMIT 25 FORMAT JSON --- pipe:list_traces:filtered:bloom [5d0533fd] +-- pipe:list_traces:filtered:bloom [022c02f1] SELECT TraceId AS traceId, Timestamp AS startTime, @@ -4522,7 +4522,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND (((has(mapKeys(SpanAttributes), 'http.method') OR has(mapKeys(SpanAttributes), 'http.request.method')) AND has(mapValues(SpanAttributes), 'GET')) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') AND ((has(mapKeys(ResourceAttributes), 'service.namespace') AND has(mapValues(ResourceAttributes), 'core')) AND ResourceAttributes['service.namespace'] = 'core') AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -4537,7 +4537,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND (((has(mapKeys(SpanAttributes), 'http.method') OR has(mapKeys(SpanAttributes), 'http.request.method')) AND has(mapValues(SpanAttributes), 'GET')) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') AND ((has(mapKeys(ResourceAttributes), 'service.namespace') AND has(mapValues(ResourceAttributes), 'core')) AND ResourceAttributes['service.namespace'] = 'core') ORDER BY ts DESC @@ -4546,7 +4546,7 @@ SELECT LIMIT 25 FORMAT JSON --- pipe:list_traces:filtered:text [e5d9f1cd] +-- pipe:list_traces:filtered:text [d0aad5c1] SELECT TraceId AS traceId, Timestamp AS startTime, @@ -4572,7 +4572,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND ((has(SpanAttributeItems, concat('http.method', char(31), 'GET')) OR has(SpanAttributeItems, concat('http.request.method', char(31), 'GET'))) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') AND (has(ResourceAttributeItems, concat('service.namespace', char(31), 'core')) AND ResourceAttributes['service.namespace'] = 'core') AND Timestamp >= (SELECT min(ts) FROM (SELECT @@ -4587,7 +4587,7 @@ SELECT AND StatusCode = 'Error' AND Duration >= 5000000 AND Duration <= 5000000000 - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND ((has(SpanAttributeItems, concat('http.method', char(31), 'GET')) OR has(SpanAttributeItems, concat('http.request.method', char(31), 'GET'))) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') AND (has(ResourceAttributeItems, concat('service.namespace', char(31), 'core')) AND ResourceAttributes['service.namespace'] = 'core') ORDER BY ts DESC @@ -7777,7 +7777,7 @@ SELECT LIMIT 50 FORMAT JSON --- spec:traces-breakdown-by-attribute:baseline [647b92c0] +-- spec:traces-breakdown-by-attribute:baseline [192e93fe] SELECT SpanAttributes['http.route'] AS name, sum(SampleRate) AS count, @@ -7795,13 +7795,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY name ORDER BY count DESC LIMIT 10 FORMAT JSON --- spec:traces-breakdown-by-attribute:bloom [647b92c0] +-- spec:traces-breakdown-by-attribute:bloom [192e93fe] SELECT SpanAttributes['http.route'] AS name, sum(SampleRate) AS count, @@ -7819,13 +7819,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY name ORDER BY count DESC LIMIT 10 FORMAT JSON --- spec:traces-breakdown-by-attribute:text [647b92c0] +-- spec:traces-breakdown-by-attribute:text [192e93fe] SELECT SpanAttributes['http.route'] AS name, sum(SampleRate) AS count, @@ -7843,13 +7843,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY name ORDER BY count DESC LIMIT 10 FORMAT JSON --- spec:traces-breakdown:baseline [35f449b0] +-- spec:traces-breakdown:baseline [948c682e] SELECT ServiceName AS name, sum(SampleRate) AS count, @@ -7867,7 +7867,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY name ORDER BY count DESC LIMIT 10 @@ -8199,7 +8199,7 @@ SELECT AND HasError = 1 FORMAT JSON --- spec:traces-list:baseline [c381db35] +-- spec:traces-list:baseline [3ec2a2e9] SELECT TraceId AS traceId, Timestamp AS timestamp, @@ -8218,7 +8218,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND Timestamp >= (SELECT min(ts) FROM (SELECT Timestamp AS ts FROM traces @@ -8226,14 +8226,14 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') ORDER BY ts DESC LIMIT 50)) ORDER BY timestamp DESC LIMIT 50 FORMAT JSON --- spec:traces-list:bloom [c381db35] +-- spec:traces-list:bloom [3ec2a2e9] SELECT TraceId AS traceId, Timestamp AS timestamp, @@ -8252,7 +8252,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND Timestamp >= (SELECT min(ts) FROM (SELECT Timestamp AS ts FROM traces @@ -8260,14 +8260,14 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') ORDER BY ts DESC LIMIT 50)) ORDER BY timestamp DESC LIMIT 50 FORMAT JSON --- spec:traces-list:text [c381db35] +-- spec:traces-list:text [3ec2a2e9] SELECT TraceId AS traceId, Timestamp AS timestamp, @@ -8286,7 +8286,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND Timestamp >= (SELECT min(ts) FROM (SELECT Timestamp AS ts FROM traces @@ -8294,7 +8294,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') ORDER BY ts DESC LIMIT 50)) ORDER BY timestamp DESC @@ -8315,7 +8315,7 @@ SELECT AND DeploymentEnv = 'production' FORMAT JSON --- spec:traces-timeseries-aggregates-mv:baseline [0cf473e6] +-- spec:traces-timeseries-aggregates-mv:baseline [cad07ec0] SELECT bucket AS bucket, groupName AS groupName, @@ -8344,7 +8344,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) GROUP BY bucket, groupName UNION ALL @@ -8368,7 +8368,7 @@ SELECT ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-all-metrics-grouped:baseline [800b31cb] +-- spec:traces-timeseries-all-metrics-grouped:baseline [bb89a9b1] SELECT toStartOfInterval(Timestamp, INTERVAL 3600 SECOND) AS bucket, coalesce(nullIf(toString(ServiceName), ''), 'all') AS groupName, @@ -8388,7 +8388,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON @@ -8769,7 +8769,7 @@ SELECT ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-apdex:baseline [7368a4c9] +-- spec:traces-timeseries-apdex:baseline [40ead943] SELECT toStartOfInterval(Timestamp, INTERVAL 300 SECOND) AS bucket, 'all' AS groupName, @@ -8789,12 +8789,12 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-attribute-filtered:baseline [b89f03f1] +-- spec:traces-timeseries-attribute-filtered:baseline [60608c7f] SELECT toStartOfInterval(Timestamp, INTERVAL 300 SECOND) AS bucket, 'all' AS groupName, @@ -8814,13 +8814,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET' GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-attribute-filtered:bloom [0ed9323f] +-- spec:traces-timeseries-attribute-filtered:bloom [fa85ef4d] SELECT toStartOfInterval(Timestamp, INTERVAL 300 SECOND) AS bucket, 'all' AS groupName, @@ -8840,13 +8840,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND (((has(mapKeys(SpanAttributes), 'http.method') OR has(mapKeys(SpanAttributes), 'http.request.method')) AND has(mapValues(SpanAttributes), 'GET')) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-attribute-filtered:text [c4e0b61b] +-- spec:traces-timeseries-attribute-filtered:text [08a8ac59] SELECT toStartOfInterval(Timestamp, INTERVAL 300 SECOND) AS bucket, 'all' AS groupName, @@ -8866,13 +8866,13 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') AND ((has(SpanAttributeItems, concat('http.method', char(31), 'GET')) OR has(SpanAttributeItems, concat('http.request.method', char(31), 'GET'))) AND if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) = 'GET') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-raw:baseline [ebfaf7db] +-- spec:traces-timeseries-raw:baseline [702c5f01] SELECT toStartOfInterval(Timestamp, INTERVAL 60 SECOND) AS bucket, 'all' AS groupName, @@ -8892,12 +8892,12 @@ SELECT AND Timestamp >= '2026-01-03 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-raw:bloom [ebfaf7db] +-- spec:traces-timeseries-raw:bloom [702c5f01] SELECT toStartOfInterval(Timestamp, INTERVAL 60 SECOND) AS bucket, 'all' AS groupName, @@ -8917,12 +8917,12 @@ SELECT AND Timestamp >= '2026-01-03 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-raw:text [ebfaf7db] +-- spec:traces-timeseries-raw:text [702c5f01] SELECT toStartOfInterval(Timestamp, INTERVAL 60 SECOND) AS bucket, 'all' AS groupName, @@ -8942,12 +8942,12 @@ SELECT AND Timestamp >= '2026-01-03 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC FORMAT JSON --- spec:traces-timeseries-series-cap:baseline [4bec4bd3] +-- spec:traces-timeseries-series-cap:baseline [3d39b07d] WITH __series_base AS ( SELECT toStartOfInterval(Timestamp, INTERVAL 300 SECOND) AS bucket, @@ -8968,7 +8968,7 @@ SELECT AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' - AND ResourceAttributes['deployment.environment'] IN ('production') + AND coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production') GROUP BY bucket, groupName ORDER BY bucket ASC, groupName ASC ) diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index bdd283e9f..5089b6610 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -509,7 +509,9 @@ describe("tracesTimeseriesQuery", () => { }) const { sql } = compileCH(q, baseParams) expect(sql).toContain("FROM traces") - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production', 'staging')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production', 'staging')", + ) }) it("filters by environments with rootOnly (MV path uses pre-extracted DeploymentEnv)", () => { @@ -608,6 +610,23 @@ describe("tracesTimeseriesQuery", () => { expect(sql).toContain("ResourceAttributes['host.name'] = 'server-1'") }) + // Either spelling of the renamed deployment-environment resource attribute + // has to match either stored key: our SDKs dual-emit, a current OTel SDK + // sends only `deployment.environment.name`, an older one only the legacy key. + for (const key of ["deployment.environment", "deployment.environment.name"]) { + it(`matches both deployment-environment spellings when filtering on ${key}`, () => { + const q = tracesTimeseriesQuery({ + metric: "count", + needsSampling: false, + resourceAttributeFilters: [{ key, value: "production", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain( + "if(ResourceAttributes['deployment.environment.name'] != '', ResourceAttributes['deployment.environment.name'], ResourceAttributes['deployment.environment']) = 'production'", + ) + }) + } + it("filters attribute with rootOnly on raw table", () => { const q = tracesTimeseriesQuery({ metric: "count", diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts index 0fb7645fb..aae572cec 100644 --- a/packages/query-engine/src/ch/queries/errors.test.ts +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -147,7 +147,9 @@ describe("errorsSummaryQuery", () => { it("applies deploymentEnvs filter", () => { const q = errorsSummaryQuery({ deploymentEnvs: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) expect(sql).toContain("FROM traces") }) }) diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 6ab5dfdf8..24908b787 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -28,6 +28,7 @@ import { matchOrIn, type FacetOutput, } from "./query-helpers" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { httpDisplaySpanName } from "../../traces-shared" import { CHNumber } from "../schema" @@ -846,7 +847,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { $.Timestamp.gte(param.dateTime("startTime")), $.Timestamp.lte(param.dateTime("endTime")), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, - CH.inList($.ResourceAttributes.get("deployment.environment"), deploymentEnvs), + CH.inList(deploymentEnvExpr($.ResourceAttributes), deploymentEnvs), ]), ) } diff --git a/packages/query-engine/src/ch/queries/infra.test.ts b/packages/query-engine/src/ch/queries/infra.test.ts index a2e646c49..cee124c80 100644 --- a/packages/query-engine/src/ch/queries/infra.test.ts +++ b/packages/query-engine/src/ch/queries/infra.test.ts @@ -119,7 +119,9 @@ describe("listPodsQuery", () => { expect(sql).toContain("ResourceAttributes['k8s.node.name'] IN") expect(sql).toContain("ResourceAttributes['k8s.cluster.name'] IN") expect(sql).toContain("ResourceAttributes['k8s.deployment.name'] IN") - expect(sql).toContain("ResourceAttributes['deployment.environment.name'] IN") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN", + ) expect(sql).toContain("'production'") }) @@ -224,7 +226,9 @@ describe("podFacetsQuery", () => { expect(sql).toContain("ResourceAttributes['k8s.statefulset.name']") expect(sql).toContain("ResourceAttributes['k8s.daemonset.name']") expect(sql).toContain("ResourceAttributes['k8s.job.name']") - expect(sql).toContain("ResourceAttributes['deployment.environment.name']") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment'])", + ) expect(sql).toContain("FORMAT JSON") }) @@ -290,7 +294,9 @@ describe("listNodesQuery", () => { baseParams, ) expect(sql).toContain("ResourceAttributes['k8s.cluster.name'] IN") - expect(sql).toContain("ResourceAttributes['deployment.environment.name'] IN") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN", + ) }) }) @@ -300,7 +306,9 @@ describe("nodeFacetsQuery", () => { expect(sql.toUpperCase().split("UNION ALL").length).toBeGreaterThan(2) expect(sql).toContain("ResourceAttributes['k8s.node.name']") expect(sql).toContain("ResourceAttributes['k8s.cluster.name']") - expect(sql).toContain("ResourceAttributes['deployment.environment.name']") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment'])", + ) }) it("scans only k8s.node.cpu.usage, not k8s.node.uptime", () => { @@ -363,7 +371,9 @@ describe("listWorkloadsQuery", () => { expect(sql).toContain("ResourceAttributes['k8s.deployment.name'] IN") expect(sql).toContain("ResourceAttributes['k8s.namespace.name'] IN") expect(sql).toContain("ResourceAttributes['k8s.cluster.name'] IN") - expect(sql).toContain("ResourceAttributes['deployment.environment.name'] IN") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN", + ) }) }) @@ -373,7 +383,9 @@ describe("workloadFacetsQuery", () => { expect(sql).toContain("ResourceAttributes['k8s.deployment.name']") expect(sql).toContain("ResourceAttributes['k8s.namespace.name']") expect(sql).toContain("ResourceAttributes['k8s.cluster.name']") - expect(sql).toContain("ResourceAttributes['deployment.environment.name']") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment'])", + ) }) it("scans only the single probe metric, not the full pod metric set", () => { diff --git a/packages/query-engine/src/ch/queries/infra.ts b/packages/query-engine/src/ch/queries/infra.ts index 37bb0c720..abdabf722 100644 --- a/packages/query-engine/src/ch/queries/infra.ts +++ b/packages/query-engine/src/ch/queries/infra.ts @@ -18,6 +18,7 @@ import { from, fromQuery, type ColumnAccessor } from "@maple-dev/clickhouse-buil import { unionAll, type CHUnionQuery, type CompiledQueryRowSchema } from "@maple-dev/clickhouse-builder" import { MetricsGauge, MetricsSum } from "../tables" import { CHNumber } from "../schema" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import type { FacetOutput } from "./query-helpers" const HOSTMETRIC_NAMES = [ @@ -405,7 +406,7 @@ const podFilterConditions = ( : undefined, opts.jobs?.length ? CH.inList($.ResourceAttributes.get("k8s.job.name"), opts.jobs) : undefined, opts.environments?.length - ? CH.inList($.ResourceAttributes.get("deployment.environment.name"), opts.environments) + ? CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments) : undefined, opts.computeTypes?.length ? CH.inList($.ResourceAttributes.get("eks.amazonaws.com/compute-type"), opts.computeTypes) @@ -432,7 +433,7 @@ export function listPodsQuery(opts: ListPodsOpts = {}) { namespace: CH.any_($.ResourceAttributes.get("k8s.namespace.name")), nodeName: CH.any_($.ResourceAttributes.get("k8s.node.name")), clusterName: CH.any_($.ResourceAttributes.get("k8s.cluster.name")), - environment: CH.any_($.ResourceAttributes.get("deployment.environment.name")), + environment: CH.any_(deploymentEnvExpr($.ResourceAttributes)), deploymentName: CH.any_($.ResourceAttributes.get("k8s.deployment.name")), statefulsetName: CH.any_($.ResourceAttributes.get("k8s.statefulset.name")), daemonsetName: CH.any_($.ResourceAttributes.get("k8s.daemonset.name")), @@ -728,7 +729,7 @@ const nodeFilterConditions = ( ? CH.inList($.ResourceAttributes.get("k8s.cluster.name"), opts.clusters) : undefined, opts.environments?.length - ? CH.inList($.ResourceAttributes.get("deployment.environment.name"), opts.environments) + ? CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments) : undefined, ] @@ -738,7 +739,7 @@ export function listNodesQuery(opts: ListNodesOpts = {}) { nodeName: $.ResourceAttributes.get("k8s.node.name"), nodeUid: CH.any_($.ResourceAttributes.get("k8s.node.uid")), clusterName: CH.any_($.ResourceAttributes.get("k8s.cluster.name")), - environment: CH.any_($.ResourceAttributes.get("deployment.environment.name")), + environment: CH.any_(deploymentEnvExpr($.ResourceAttributes)), kubeletVersion: CH.any_($.ResourceAttributes.get("k8s.kubelet.version")), lastSeen: CH.max_($.TimeUnix), cpuUsage: CH.avgIf($.Value, $.MetricName.eq("k8s.node.cpu.usage")), @@ -861,7 +862,7 @@ const workloadFilterConditions = ( ? CH.inList($.ResourceAttributes.get("k8s.cluster.name"), opts.clusters) : undefined, opts.environments?.length - ? CH.inList($.ResourceAttributes.get("deployment.environment.name"), opts.environments) + ? CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments) : undefined, opts.computeTypes?.length ? CH.inList($.ResourceAttributes.get("eks.amazonaws.com/compute-type"), opts.computeTypes) @@ -875,7 +876,7 @@ export function listWorkloadsQuery(opts: ListWorkloadsOpts) { workloadName: $.ResourceAttributes.get(attrKey), namespace: CH.any_($.ResourceAttributes.get("k8s.namespace.name")), clusterName: CH.any_($.ResourceAttributes.get("k8s.cluster.name")), - environment: CH.any_($.ResourceAttributes.get("deployment.environment.name")), + environment: CH.any_(deploymentEnvExpr($.ResourceAttributes)), podCount: CH.uniq($.ResourceAttributes.get("k8s.pod.uid")), lastSeen: CH.max_($.TimeUnix), avgCpuLimitPct: CH.avgIf($.Value, $.MetricName.eq("k8s.pod.cpu_limit_utilization")), @@ -976,19 +977,35 @@ export function workloadGaugeTimeseriesQuery(opts: WorkloadGaugeTimeseriesOpts) // list query (pods, nodes, or workloads), filtered by the same opts so the // facet counts reflect the *current* filtered set. +/** + * Facet dimensions are plain `ResourceAttributes` keys, with one exception: the + * deployment environment has two spellings in the wild (OTel renamed + * `deployment.environment` to `deployment.environment.name`), so it resolves to + * the coalescing expression instead of a single map lookup. Without this a + * cluster whose collector still emits the legacy key produced an empty + * environment facet. + */ +const facetAttrExpr = ( + resourceAttributes: { get(key: string): CH.Expr }, + attrKey: string, +): CH.Expr => + attrKey === "deployment.environment.name" + ? deploymentEnvExpr(resourceAttributes) + : resourceAttributes.get(attrKey) + export type PodFacetsOutput = FacetOutput const makePodFacet = (opts: ListPodsOpts, attrKey: string, facetType: string, perFacetLimit: number) => from(MetricsGauge) .select(($) => ({ - name: $.ResourceAttributes.get(attrKey), + name: facetAttrExpr($.ResourceAttributes, attrKey), count: CH.uniq($.ResourceAttributes.get("k8s.pod.uid")), facetType: CH.lit(facetType), })) .where(($) => [ ...podBaseConditions($, [POD_FACET_PROBE_METRIC]), ...podFilterConditions($, opts), - $.ResourceAttributes.get(attrKey).neq(""), + facetAttrExpr($.ResourceAttributes, attrKey).neq(""), ]) .groupBy("name") .orderBy(["count", "desc"]) @@ -1014,14 +1031,14 @@ export type NodeFacetsOutput = FacetOutput const makeNodeFacet = (opts: ListNodesOpts, attrKey: string, facetType: string, perFacetLimit: number) => from(MetricsGauge) .select(($) => ({ - name: $.ResourceAttributes.get(attrKey), + name: facetAttrExpr($.ResourceAttributes, attrKey), count: CH.uniq($.ResourceAttributes.get("k8s.node.name")), facetType: CH.lit(facetType), })) .where(($) => [ ...nodeBaseConditions($, [NODE_FACET_PROBE_METRIC]), ...nodeFilterConditions($, opts), - $.ResourceAttributes.get(attrKey).neq(""), + facetAttrExpr($.ResourceAttributes, attrKey).neq(""), ]) .groupBy("name") .orderBy(["count", "desc"]) @@ -1046,7 +1063,7 @@ const makeWorkloadFacet = ( const ownerKey = workloadAttrKey(opts.kind) return from(MetricsGauge) .select(($) => ({ - name: $.ResourceAttributes.get(attrKey), + name: facetAttrExpr($.ResourceAttributes, attrKey), count: CH.uniq($.ResourceAttributes.get(ownerKey)), facetType: CH.lit(facetType), })) @@ -1057,7 +1074,7 @@ const makeWorkloadFacet = ( $.ResourceAttributes.get(ownerKey).neq(""), $.MetricName.in_(POD_FACET_PROBE_METRIC), ...workloadFilterConditions($, opts, ownerKey), - $.ResourceAttributes.get(attrKey).neq(""), + facetAttrExpr($.ResourceAttributes, attrKey).neq(""), ]) .groupBy("name") .orderBy(["count", "desc"]) diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts index 26737e4f7..4e41e9324 100644 --- a/packages/query-engine/src/ch/queries/logs.test.ts +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -234,7 +234,9 @@ describe("logsTimeseriesQuery MV routing", () => { const { sql } = compileCH(q, baseParams) expect(sql).toContain("FROM logs") expect(sql).not.toContain("logs_aggregates_hourly") - expect(sql).toContain("positionCaseInsensitive(ResourceAttributes['deployment.environment'], 'prod')") + expect(sql).toContain( + "positionCaseInsensitive(coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']), 'prod')", + ) }) }) @@ -279,7 +281,9 @@ describe("logsBreakdownQuery", () => { const { sql } = compileCH(q, baseParams) expect(sql).toContain("FROM logs") expect(sql).not.toContain("logs_aggregates_hourly") - expect(sql).toContain("positionCaseInsensitive(ResourceAttributes['deployment.environment'], 'prod')") + expect(sql).toContain( + "positionCaseInsensitive(coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']), 'prod')", + ) }) it("applies custom limit", () => { @@ -581,7 +585,9 @@ describe("logsFacetsQuery", () => { const { sql } = compileUnion(q, baseParams) expect(sql).toContain("FROM logs") expect(sql).not.toContain("logs_aggregates_hourly") - expect(sql).toContain("positionCaseInsensitive(ResourceAttributes['deployment.environment'], 'prod')") + expect(sql).toContain( + "positionCaseInsensitive(coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']), 'prod')", + ) }) it("compiles only the requested branch when facet is set", () => { @@ -615,31 +621,41 @@ describe("environments filter", () => { it("logsListQuery filters by a single environment", () => { const q = logsListQuery({ environments: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) }) it("logsListQuery filters by multiple environments", () => { const q = logsListQuery({ environments: ["production", "staging"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production', 'staging')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production', 'staging')", + ) }) it("logsListQuery uses positionCaseInsensitive for single-value contains mode", () => { const q = logsListQuery({ environments: ["prod"], matchModes: { deploymentEnv: "contains" } }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("positionCaseInsensitive(ResourceAttributes['deployment.environment'], 'prod')") + expect(sql).toContain( + "positionCaseInsensitive(coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']), 'prod')", + ) }) it("logsCountQuery applies environments filter", () => { const q = logsCountQuery({ environments: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) }) it("logsTimeseriesQuery applies environments filter", () => { const q = logsTimeseriesQuery({ environments: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) }) it("logsFacetsQuery applies environments filter to all branches via the MV column", () => { diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 7561fc6d0..01c36082f 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -12,6 +12,7 @@ import { unionAll, type CHUnionQuery } from "@maple-dev/clickhouse-builder" import { Logs, LogsAggregatesHourly } from "../tables" import { finalizeTimeseries } from "./series-cap" import type { AttributeFilter } from "@maple/domain/query-engine" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition } from "../../traces-shared" import type { AttributeIndexMode, LogBodySearchMode } from "../../capabilities" import { edgeCondition, interiorConditions } from "./rollup-splice" @@ -113,7 +114,7 @@ function environmentCondition( opts: LogsQueryOpts, ): CH.Condition | undefined { if (!opts.environments?.length) return undefined - const envAttr = $.ResourceAttributes.get("deployment.environment") + const envAttr = deploymentEnvExpr($.ResourceAttributes) if (opts.matchModes?.deploymentEnv === "contains" && opts.environments.length === 1) { return CH.positionCaseInsensitive(envAttr, CH.lit(opts.environments[0]!)).gt(0) } @@ -846,12 +847,12 @@ function logsFacetsQueryFromRaw( .select(($) => ({ severityText: CH.lit(""), serviceName: CH.lit(""), - deploymentEnv: $.ResourceAttributes.get("deployment.environment"), + deploymentEnv: deploymentEnvExpr($.ResourceAttributes), namespace: CH.lit(""), count: CH.count(), facetType: CH.lit("deploymentEnv"), })) - .where(($) => [...baseWhere($), $.ResourceAttributes.get("deployment.environment").neq("")]) + .where(($) => [...baseWhere($), deploymentEnvExpr($.ResourceAttributes).neq("")]) .groupBy("deploymentEnv") const namespaceQuery = from(Logs) diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts index 162cc560d..00c19eb47 100644 --- a/packages/query-engine/src/ch/queries/metrics.test.ts +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -40,7 +40,9 @@ describe("metricsTimeseriesQuery", () => { // scope reads the resource-attribute map directly. const q = metricsTimeseriesQuery({ metricType: "sum", environments: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) }) it("omits the environment predicate when no environments are selected", () => { @@ -253,7 +255,9 @@ describe("metricsTimeseriesRateQuery", () => { it("applies an environment filter in the CTE", () => { const q = metricsTimeseriesRateQuery({ environments: ["production", "staging"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production', 'staging')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production', 'staging')", + ) }) it("falls back to raw metrics_sum when an environment filter is set", () => { diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index b19908065..da27f643c 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -12,6 +12,7 @@ import { table } from "@maple-dev/clickhouse-builder" import { MetricsSum, MetricCatalog, SpanMetricsCallsHourly } from "../tables" import { compileCH } from "@maple-dev/clickhouse-builder" import { resolveMetricTable, metricsSelectExprs } from "./query-helpers" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition } from "../../traces-shared" import { finalizeTimeseries } from "./series-cap" @@ -27,14 +28,6 @@ function resourceFilterConditions( return (filters ?? []).map((rf) => buildAttrFilterCondition(rf, "ResourceAttributes")) } -/** - * Resource-attribute key holding the deployment environment. Metrics tables - * carry no pre-extracted `DeploymentEnv` column (unlike the trace MVs), so an - * environment filter reads the map directly — the same key - * `tracesBaseWhereConditions` uses on the raw `traces` table. - */ -const DEPLOYMENT_ENV_KEY = "deployment.environment" - // Shared options & output types interface MetricsQueryOpts { @@ -111,7 +104,7 @@ export function metricsTimeseriesQuery(opts: MetricsTimeseriesOpts) { CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.attributeKey, (k: string) => $.Attributes.get(k).eq(opts.attributeValue ?? "")), opts.environments?.length - ? CH.inList($.ResourceAttributes.get(DEPLOYMENT_ENV_KEY), opts.environments) + ? CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments) : undefined, ...resourceFilterConditions(opts.resourceAttributeFilters), ]) @@ -384,7 +377,7 @@ export function metricsTimeseriesRateQuery( CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.attributeKey, (k: string) => $.Attributes.get(k).eq(opts.attributeValue ?? "")), opts.environments?.length - ? CH.inList($.ResourceAttributes.get(DEPLOYMENT_ENV_KEY), opts.environments) + ? CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments) : undefined, ...resourceFilterConditions(opts.resourceAttributeFilters), ]), diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index 0c80bad92..8a28e9980 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -9,6 +9,7 @@ import { param } from "@maple-dev/clickhouse-builder" import type { ColumnAccessor } from "@maple-dev/clickhouse-builder" import type { ServiceOverviewSpans, Traces, TracesAggregatesHourly } from "../tables" import { MetricsSum, MetricsGauge, MetricsHistogram, MetricsExpHistogram } from "../tables" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition, httpDisplaySpanName } from "../../traces-shared" import type { AttributeIndexMode } from "../../capabilities" @@ -254,12 +255,12 @@ export function tracesBaseWhereConditions( if (mm?.deploymentEnv === "contains" && opts.environments.length === 1) { conditions.push( CH.positionCaseInsensitive( - $.ResourceAttributes.get("deployment.environment"), + deploymentEnvExpr($.ResourceAttributes), CH.lit(opts.environments[0]), ).gt(0), ) } else { - conditions.push(CH.inList($.ResourceAttributes.get("deployment.environment"), opts.environments)) + conditions.push(CH.inList(deploymentEnvExpr($.ResourceAttributes), opts.environments)) } } if (opts.namespaces?.length) { @@ -304,9 +305,7 @@ export function tracesBaseWhereConditions( ) } if (opts.excludedEnvironments?.length) { - conditions.push( - CH.notInList($.ResourceAttributes.get("deployment.environment"), opts.excludedEnvironments), - ) + conditions.push(CH.notInList(deploymentEnvExpr($.ResourceAttributes), opts.excludedEnvironments)) } if (opts.excludedNamespaces?.length) { conditions.push(CH.notInList($.ResourceAttributes.get("service.namespace"), opts.excludedNamespaces)) diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.ts b/packages/query-engine/src/ch/queries/service-map-rollup.ts index 28b7c30b0..637691808 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.ts @@ -19,6 +19,7 @@ import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, fromQuery } from "@maple-dev/clickhouse-builder" import { ServiceAddressResolutionsHourly, ServiceMapEdgesHourly, Traces } from "../tables" +import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { serviceMapEdgeJoinQuery } from "./service-map" import { CHNumber } from "../schema" @@ -216,7 +217,7 @@ export function serviceMapResolutionsRollupSQL( SpanId: $.SpanId, ServiceName: $.ServiceName, ServerAddress: $.SpanAttributes.get("server.address"), - DeploymentEnv: $.ResourceAttributes.get("deployment.environment"), + DeploymentEnv: deploymentEnvExpr($.ResourceAttributes), })) .where(($) => [ CH.inList($.SpanKind, ["Client", "Producer"]), diff --git a/packages/query-engine/src/ch/queries/service-map.test.ts b/packages/query-engine/src/ch/queries/service-map.test.ts index c272b486b..40370a70a 100644 --- a/packages/query-engine/src/ch/queries/service-map.test.ts +++ b/packages/query-engine/src/ch/queries/service-map.test.ts @@ -109,7 +109,9 @@ describe("serviceExternalEdgesSQL", () => { baseParams, ) expect(sql).toContain("DeploymentEnv = 'production'") - expect(sql).toContain("ResourceAttributes['deployment.environment'] = 'production'") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) = 'production'", + ) }) it("groups by target identity and orders by callCount desc", () => { @@ -536,7 +538,9 @@ describe("serviceDbEdgesForServiceQuery", () => { baseParams, ) expect(sql).toContain("DeploymentEnv = 'production'") - expect(sql).toContain("ResourceAttributes['deployment.environment'] = 'production'") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) = 'production'", + ) }) it("orders by callCount desc, limits to 200, formats as JSON", () => { @@ -579,7 +583,9 @@ describe("service-map database query summaries", () => { expect(sql).toContain("Timestamp <= toDateTime('2024-01-02 00:00:00')") expect(sql).toContain("OrgId = 'org_1'") expect(sql).toContain("ServiceName = 'artifacts-api'") - expect(sql).toContain("ResourceAttributes['deployment.environment'] = 'production'") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) = 'production'", + ) expect(sql).toContain( "coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) = 'postgresql'", ) @@ -706,7 +712,9 @@ describe("service-map database query summaries", () => { }) expect(sql).toContain("= 'post\\'gres'") expect(sql).toContain("ServiceName = 'svc\\'one'") - expect(sql).toContain("ResourceAttributes['deployment.environment'] = 'prod\\'west'") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) = 'prod\\'west'", + ) }) it.effect("decodes summary rows with numeric strings from ClickHouse JSON", () => diff --git a/packages/query-engine/src/ch/queries/service-map.ts b/packages/query-engine/src/ch/queries/service-map.ts index cf408e912..6408967e0 100644 --- a/packages/query-engine/src/ch/queries/service-map.ts +++ b/packages/query-engine/src/ch/queries/service-map.ts @@ -14,6 +14,7 @@ import { OPAQUE_DB_NAMESPACE_RE, presentableStatementSql, } from "@maple/domain/tinybird/db-query-shape-sql" +import { deploymentEnvExpr, messagingDestinationExpr } from "@maple/domain/tinybird/semconv-renames" import { Schema } from "effect" import { compileCH, type CompiledQuery, type CompiledQueryRowSchema } from "@maple-dev/clickhouse-builder" import { defineCondFn, defineFn } from "@maple-dev/clickhouse-builder" @@ -508,9 +509,7 @@ function serviceDbEdgesQueryBase(opts: { serviceName?: string; deploymentEnv?: s $.Timestamp.lte(param.dateTime("endTime")), $.SpanKind.in_("Client", "Producer"), dbSystemExpr($).neq(""), - opts.deploymentEnv - ? $.ResourceAttributes.get("deployment.environment").eq(opts.deploymentEnv) - : undefined, + opts.deploymentEnv ? deploymentEnvExpr($.ResourceAttributes).eq(opts.deploymentEnv) : undefined, ]) .groupBy("sourceService", "dbSystem", "dbNamespace") @@ -736,9 +735,7 @@ const serviceDbRawFilters = ( // Same undefined-vs-'' semantics as `shapesHourlyFilters`. params.dbNamespace !== undefined ? dbNamespaceExpr($).eq(params.dbNamespace) : undefined, params.sourceService ? $.ServiceName.eq(params.sourceService) : undefined, - params.deploymentEnv - ? $.ResourceAttributes.get("deployment.environment").eq(params.deploymentEnv) - : undefined, + params.deploymentEnv ? deploymentEnvExpr($.ResourceAttributes).eq(params.deploymentEnv) : undefined, ] // Aggregate-state expressions shared by the summary/timeseries/top-queries @@ -1058,7 +1055,8 @@ export function serviceExternalEdgesSQL( const recentEdges = from(Traces) .select(($) => { const attr = (key: string) => $.SpanAttributes.get(key) - const isMessaging = attr("messaging.destination").neq("").or(attr("messaging.system").neq("")) + const destination = messagingDestinationExpr($.SpanAttributes) + const isMessaging = destination.neq("").or(attr("messaging.system").neq("")) const isRpc = attr("rpc.service").neq("").or(attr("rpc.system").neq("")) return { sourceService: $.ServiceName, @@ -1078,14 +1076,7 @@ export function serviceExternalEdgesSQL( ), targetName: CH.multiIf( [ - [ - isMessaging, - CH.if_( - attr("messaging.destination").neq(""), - attr("messaging.destination"), - attr("messaging.system"), - ), - ], + [isMessaging, CH.if_(destination.neq(""), destination, attr("messaging.system"))], [isRpc, CH.if_(attr("rpc.service").neq(""), attr("rpc.service"), attr("rpc.system"))], ], CH.if_( @@ -1114,12 +1105,12 @@ export function serviceExternalEdgesSQL( .neq("") .or(attr("http.host").neq("")) .or(attr("url.authority").neq("")) - .or(attr("messaging.destination").neq("")) + .or(messagingDestinationExpr($.SpanAttributes).neq("")) .or(attr("messaging.system").neq("")) .or(attr("rpc.service").neq("")) .or(attr("rpc.system").neq("")), opts.deploymentEnv - ? $.ResourceAttributes.get("deployment.environment").eq(opts.deploymentEnv) + ? deploymentEnvExpr($.ResourceAttributes).eq(opts.deploymentEnv) : undefined, ] }) diff --git a/packages/query-engine/src/ch/queries/service-operations.test.ts b/packages/query-engine/src/ch/queries/service-operations.test.ts index 3f4587aa5..5ca4dfdd9 100644 --- a/packages/query-engine/src/ch/queries/service-operations.test.ts +++ b/packages/query-engine/src/ch/queries/service-operations.test.ts @@ -77,7 +77,9 @@ describe("serviceOperationsSummaryQuery", () => { it("applies environment filter via ResourceAttributes", () => { const q = serviceOperationsSummaryQuery({ serviceName: "api", environments: ["production"] }) const { sql } = compileCH(q, baseParams) - expect(sql).toContain("ResourceAttributes['deployment.environment'] IN ('production')") + expect(sql).toContain( + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) expect(sql).toContain("DeploymentEnv IN ('production')") }) diff --git a/packages/query-engine/src/runtime/metrics-alert-evaluation.test.ts b/packages/query-engine/src/runtime/metrics-alert-evaluation.test.ts index d77ccf630..07acea89d 100644 --- a/packages/query-engine/src/runtime/metrics-alert-evaluation.test.ts +++ b/packages/query-engine/src/runtime/metrics-alert-evaluation.test.ts @@ -74,7 +74,10 @@ describe("metric alert evaluation", () => { ) assert.include(sql, "OrgId = 'org_metrics_alert_test'") - assert.include(sql, "ResourceAttributes['deployment.environment'] IN ('production')") + assert.include( + sql, + "coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) IN ('production')", + ) assert.strictEqual(context, "metricsAlertEval") assert.deepStrictEqual(result, [ { diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index a61faf874..dbac9f21f 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -31,6 +31,7 @@ export const TRACE_LIST_MV_ATTR_MAP: Record = { export const TRACE_LIST_MV_RESOURCE_MAP: Record = { "deployment.environment": "DeploymentEnv", + "deployment.environment.name": "DeploymentEnv", } satisfies Record // Attribute filter → typed Condition @@ -38,7 +39,7 @@ export const TRACE_LIST_MV_RESOURCE_MAP: Record = { import * as CH from "@maple-dev/clickhouse-builder/expr" import { normalizedSpanNameExpr } from "@maple/domain/tinybird/span-display-name" -// HTTP semconv coalescing +// Semconv rename coalescing // // OpenTelemetry renamed several HTTP span attributes in the stable semconv: // http.method → http.request.method @@ -56,6 +57,19 @@ const HTTP_SEMCONV_ALIASES: Record = { "http.response.status_code": ["http.status_code", "http.response.status_code"], } satisfies Record +/** + * The same treatment for the one renamed *resource* attribute: the registry + * marks `deployment.environment` deprecated, "Replaced by + * `deployment.environment.name`". Both spellings are in the wild — our own SDKs + * dual-emit, a current OTel SDK sends only `.name`, an older one only the legacy + * key — so either filter spelling has to match either stored key. Canonical + * first, matching `DEPLOYMENT_ENV_SQL`'s coalesce order. + */ +const RESOURCE_SEMCONV_ALIASES: Record = { + "deployment.environment": ["deployment.environment.name", "deployment.environment"], + "deployment.environment.name": ["deployment.environment.name", "deployment.environment"], +} satisfies Record + /** `if(map[k0] != '', map[k0], if(map[k1] != '', …))` — first non-empty alias. */ function coalescedMapGet(mapExpr: CH.Expr>, keys: readonly string[]): CH.Expr { let expr = CH.mapGet(mapExpr, keys[keys.length - 1]) @@ -95,9 +109,16 @@ export function buildAttrFilterCondition( indexMode: AttributeIndexMode = "none", ): CH.Condition { const mapExpr = CH.dynamicColumn>(mapName) - // Span attributes renamed across OTel semconv versions match either spelling, - // mirroring trace_list_mv. Resource attributes have no such aliases. - const keys = mapName === "SpanAttributes" ? (HTTP_SEMCONV_ALIASES[af.key] ?? [af.key]) : [af.key] + // Attributes renamed across OTel semconv versions match either spelling, + // mirroring trace_list_mv (span attributes) and the MVs' pre-extracted + // `DeploymentEnv` (resource attributes). + const aliasTable = + mapName === "SpanAttributes" + ? HTTP_SEMCONV_ALIASES + : mapName === "ResourceAttributes" + ? RESOURCE_SEMCONV_ALIASES + : undefined + const keys = aliasTable?.[af.key] ?? [af.key] const colExpr: CH.Expr = coalescedMapGet(mapExpr, keys) const value = af.value ?? ""