Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/reference/configuration.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,26 @@ Override the machine hostname sent with events.

Useful in containerized environments where the system hostname is a random container ID. Set this to a stable identifier like the pod name or instance ID.

### `pg_stat_ch.extra_attributes`

Exporter metadata expressed as semicolon-separated `key:value` pairs. `instance_uuid` is included in all export formats; the other resource columns are populated by the Arrow exporters.

| | |
|---|---|
| **Type** | string |
| **Default** | `''` |
| **Context** | sighup |

Supply `instance_uuid` with the Postgres service's canonical UUID alongside the existing `instance_ubid` and server attributes:

```ini
pg_stat_ch.extra_attributes = 'instance_uuid:01234567-89ab-8ad0-9234-56789abcdef0;instance_ubid:pg04hmasw9ne4j8t5cy4tqkff1;server_role:primary'
```

The provisioner supplies the UUID; the producer does not derive it from the UBID. Native ClickHouse and Arrow exports include an `instance_uuid` string column, and ordinary OTLP exports include an `instance_uuid` log attribute. An omitted value produces an empty string.

The bundled ClickHouse migration adds `instance_uuid String DEFAULT ''` to `events_raw`. Apply it **before deploying the producer when using native ClickHouse export**: its named inserts require the column even when the UUID is unset. ArrowStream inserts can arrive before the column exists; ClickHouse ignores the extra field until the schema includes it. External schemas and materialized-view projections need their own updates to retain the UUID; previously ignored values are not backfilled.

### `pg_stat_ch.log_min_elevel`

Minimum error severity level to capture via the `emit_log_hook`.
Expand Down
1 change: 1 addition & 0 deletions docs/reference/events-schema.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ The table is partitioned by date (`toDate(ts_start)`) and ordered by `ts_start`

| Column | Type | Description |
|---|---|---|
| `instance_uuid` | `String` | Postgres service UUID supplied through `pg_stat_ch.extra_attributes`. Native ClickHouse and both Arrow exporters include this field; OTLP emits it as a log attribute. It is empty when unset. |
| `db` | `LowCardinality(String)` | PostgreSQL database name. |
| `username` | `LowCardinality(String)` | PostgreSQL user or role that executed the query. |
| `pid` | `Int32` | Backend process ID. Correlate with `pg_stat_activity` for session-level debugging. |
Expand Down
10 changes: 10 additions & 0 deletions schema/migrations/20260925000001_add_instance_uuid.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- +goose Up
ALTER TABLE pg_stat_ch.events_raw
ADD COLUMN IF NOT EXISTS instance_uuid String DEFAULT ''
COMMENT 'Postgres service UUID supplied by pg_stat_ch.extra_attributes; empty when unset.'
AFTER instance_ubid;

-- +goose Down
-- Retain the additive column on rollback: events_recent_1h's SELECT * depends
-- on it, and older producers can still insert without supplying it.
SELECT 1;
5 changes: 3 additions & 2 deletions src/config/guc.c
Original file line number Diff line number Diff line change
Expand Up @@ -389,8 +389,9 @@ void PschInitGuc(void) {

DefineCustomStringVariable(
"pg_stat_ch.extra_attributes",
"Key-value pairs appended to exported Arrow batches.",
"Semicolon-separated k:v pairs for resource columns: "
"Key-value metadata for exported events.",
"Semicolon-separated k:v pairs for resource columns, including instance_uuid "
"for the Postgres service UUID in all export formats: "
"'instance_ubid:abc;server_role:primary;read_replica_type:regional;region:us-east-1'.",
&psch_extra_attributes,
"",
Expand Down
20 changes: 13 additions & 7 deletions src/export/arrow_batch.cc
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ struct ArrowBatchBuilder::Impl {
arrow::UInt32Builder parallel_workers_planned_builder;
arrow::UInt32Builder parallel_workers_launched_builder;
arrow::StringBuilder instance_ubid_builder;
arrow::StringBuilder instance_uuid_builder;
arrow::StringBuilder server_ubid_builder;
DictBuilder server_role_builder;
DictBuilder read_replica_type_builder;
Expand Down Expand Up @@ -259,6 +260,7 @@ struct ArrowBatchBuilder::Impl {
arrow::field("parallel_workers_planned", arrow::uint32()),
arrow::field("parallel_workers_launched", arrow::uint32()),
arrow::field("instance_ubid", arrow::utf8()),
arrow::field("instance_uuid", arrow::utf8()),
arrow::field("server_ubid", arrow::utf8()),
arrow::field("server_role", DictionaryUtf8Type()),
arrow::field("read_replica_type", DictionaryUtf8Type()),
Expand Down Expand Up @@ -424,6 +426,8 @@ struct ArrowBatchBuilder::Impl {

if (!AppendString(&instance_ubid_builder, ExtraAttr("instance_ubid"),
"Arrow instance_ubid append") ||
!AppendString(&instance_uuid_builder, ExtraAttr("instance_uuid"),
"Arrow instance_uuid append") ||
!AppendString(&server_ubid_builder, ExtraAttr("server_ubid"), "Arrow server_ubid append") ||
!AppendString(&server_role_builder, ExtraAttr("server_role"), "Arrow server_role append") ||
!AppendString(&read_replica_type_builder, ExtraAttr("read_replica_type"),
Expand All @@ -436,13 +440,13 @@ struct ArrowBatchBuilder::Impl {
return false;
}

estimated_bytes += kFixedBytesPerRow + db_name.size() + db_user.size() + app.size() +
client_addr.size() + query_text.size() + err_message.size() +
err_sqlstate.size() + service_version.size() +
ExtraAttr("instance_ubid").size() + ExtraAttr("server_ubid").size() +
ExtraAttr("server_role").size() + ExtraAttr("read_replica_type").size() +
ExtraAttr("region").size() + ExtraAttr("cell").size() +
ExtraAttr("host_id").size() + ExtraAttr("pod_name").size();
estimated_bytes +=
kFixedBytesPerRow + db_name.size() + db_user.size() + app.size() + client_addr.size() +
query_text.size() + err_message.size() + err_sqlstate.size() + service_version.size() +
ExtraAttr("instance_ubid").size() + ExtraAttr("instance_uuid").size() +
ExtraAttr("server_ubid").size() + ExtraAttr("server_role").size() +
ExtraAttr("read_replica_type").size() + ExtraAttr("region").size() +
ExtraAttr("cell").size() + ExtraAttr("host_id").size() + ExtraAttr("pod_name").size();
++num_rows;
return true;
}
Expand Down Expand Up @@ -522,6 +526,7 @@ struct ArrowBatchBuilder::Impl {
!add_array(&parallel_workers_planned_builder, "Arrow parallel_workers_planned finish") ||
!add_array(&parallel_workers_launched_builder, "Arrow parallel_workers_launched finish") ||
!add_array(&instance_ubid_builder, "Arrow instance_ubid finish") ||
!add_array(&instance_uuid_builder, "Arrow instance_uuid finish") ||
!add_array(&server_ubid_builder, "Arrow server_ubid finish") ||
!add_dict_array(&server_role_builder, "Arrow server_role finish") ||
!add_dict_array(&read_replica_type_builder, "Arrow read_replica_type finish") ||
Expand Down Expand Up @@ -629,6 +634,7 @@ struct ArrowBatchBuilder::Impl {
parallel_workers_planned_builder.Reset();
parallel_workers_launched_builder.Reset();
instance_ubid_builder.Reset();
instance_uuid_builder.Reset();
server_ubid_builder.Reset();
server_role_builder.ResetFull();
read_replica_type_builder.ResetFull();
Expand Down
46 changes: 46 additions & 0 deletions src/export/extra_attributes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#ifndef PG_STAT_CH_SRC_EXPORT_EXTRA_ATTRIBUTES_H_
#define PG_STAT_CH_SRC_EXPORT_EXTRA_ATTRIBUTES_H_

#include <string>
#include <string_view>
#include <utility>
#include <vector>

// Parse "key1:val1;key2:val2" into a flat list. First match wins on
// duplicate keys (Get linear-scans from the front). Empty input -> empty list.
class ExtraAttrs {
public:
explicit ExtraAttrs(const char* raw) {
if (raw == nullptr) {
return;
}
std::string_view input(raw);
while (!input.empty()) {
const size_t delim = input.find(';');
const std::string_view token =
(delim == std::string_view::npos) ? input : input.substr(0, delim);
const size_t sep = token.find(':');
if (sep != std::string_view::npos) {
attrs_.emplace_back(std::string(token.substr(0, sep)), std::string(token.substr(sep + 1)));
}
if (delim == std::string_view::npos) {
break;
}
input.remove_prefix(delim + 1);
}
}

std::string Get(std::string_view key) const {
for (const auto& [k, v] : attrs_) {
if (k == key) {
return v;
}
}
return {};
}

private:
std::vector<std::pair<std::string, std::string>> attrs_;
};

#endif // PG_STAT_CH_SRC_EXPORT_EXTRA_ATTRIBUTES_H_
40 changes: 2 additions & 38 deletions src/export/otel_arrow_exporter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ extern "C" {
#include "pg_stat_ch/pg_stat_ch.h"
#include "config/guc.h"
#include "export/exporter_interface.h"
#include "export/extra_attributes.h"
#include "export/otel_arrow_exporter.h"
#include "export/otel_exporter.h"

Expand Down Expand Up @@ -117,43 +118,6 @@ struct ArrowSlot {
std::shared_ptr<arrow::ArrayBuilder> builder;
};

// Parse "key1:val1;key2:val2" into a flat list. First match wins on
// duplicate keys (Get linear-scans from the front). Empty input -> empty list.
class ExtraAttrs {
public:
explicit ExtraAttrs(const char* raw) {
if (raw == nullptr) {
return;
}
std::string_view input(raw);
while (!input.empty()) {
const size_t delim = input.find(';');
const std::string_view token =
(delim == std::string_view::npos) ? input : input.substr(0, delim);
const size_t sep = token.find(':');
if (sep != std::string_view::npos) {
attrs_.emplace_back(std::string(token.substr(0, sep)), std::string(token.substr(sep + 1)));
}
if (delim == std::string_view::npos) {
break;
}
input.remove_prefix(delim + 1);
}
}

std::string Get(std::string_view key) const {
for (const auto& [k, v] : attrs_) {
if (k == key) {
return v;
}
}
return {};
}

private:
std::vector<std::pair<std::string, std::string>> attrs_;
};

// ---------------------------------------------------------------------------

class OTelArrowExporter : public StatsExporter {
Expand Down Expand Up @@ -371,7 +335,7 @@ class OTelArrowExporter : public StatsExporter {
// BeginRow so stats_exporter.cc's column-emission loop doesn't have to
// know about them:
//
// - 8 envelope columns + read_replica_type: per-process constants from
// - Envelope columns: per-process constants from
// pg_stat_ch.extra_attributes (or "none" default for read_replica_type
// per clickgres-platform's convention).
// - service_version: PG_STAT_CH_VERSION macro, not from extra_attributes.
Expand Down
4 changes: 4 additions & 0 deletions src/export/stats_exporter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ extern "C" {
#include "export/arrow_batch.h"
#include "export/clickhouse_exporter.h"
#include "export/exporter_interface.h"
#include "export/extra_attributes.h"
#include "export/otel_arrow_exporter.h"
#include "export/otel_exporter.h"
#include "export/stats_exporter.h"
Expand Down Expand Up @@ -239,6 +240,8 @@ void ExportEventStatsInternal(const std::vector<PschEvent>& events, StatsExporte

exporter->BeginBatch();

const std::string instance_uuid = ExtraAttrs(psch_extra_attributes).Get("instance_uuid");
auto col_instance_uuid = exporter->StatHCString("instance_uuid");
auto col_ts = exporter->StatTimestamp("ts");
auto col_duration_us = exporter->DbDurationColumn();
auto col_db_name = exporter->DbNameColumn();
Expand Down Expand Up @@ -294,6 +297,7 @@ void ExportEventStatsInternal(const std::vector<PschEvent>& events, StatsExporte
for (const auto& ev : events) {
exporter->BeginRow();

col_instance_uuid->Append(instance_uuid);
col_ts->Append(ev.ts_start + kPostgresEpochOffsetUs);
col_duration_us->Append(ev.duration_us);
col_db_name->Append(std::string(ev.datname, ev.datname_len));
Expand Down
16 changes: 16 additions & 0 deletions t/010_clickhouse_export.pl
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@
10
);
cmp_ok($query_check, '>=', 1, 'Query text is captured');

my $uuid_count = psch_query_clickhouse(
"SELECT count() FROM pg_stat_ch.events_raw WHERE instance_uuid != ''");
chomp $uuid_count;
is($uuid_count, '0', 'Unconfigured instance UUID is empty');
};

# Test 2: Batch sizing - verify batch_max is honored
Expand Down Expand Up @@ -112,6 +117,10 @@

# Test 4: All fields populated
subtest 'all fields populated' => sub {
my $instance_uuid = '01234567-89ab-8ad0-9234-56789abcdef0';
$node->safe_psql('postgres',
"ALTER SYSTEM SET pg_stat_ch.extra_attributes = 'instance_uuid:$instance_uuid'");
$node->restart();
psch_query_clickhouse("TRUNCATE TABLE pg_stat_ch.events_raw");
psch_reset_stats($node);

Expand Down Expand Up @@ -147,6 +156,13 @@
);
cmp_ok($db_operation_check, '>=', 1, 'db_operation is populated');

my $uuid_check = psch_wait_for_clickhouse_query(
"SELECT count() FROM pg_stat_ch.events_raw WHERE instance_uuid = '$instance_uuid'",
sub { $_[0] >= 1 },
10
);
cmp_ok($uuid_check, '>=', 1, 'Configured instance UUID is exported natively');

# Clean up
$node->safe_psql('postgres', 'DROP TABLE IF EXISTS test_fields');
};
Expand Down
Loading
Loading