From 023cdefcb59c6e9a10255a0178be6b7924cb3a83 Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Wed, 18 Nov 2026 05:05:56 +0000 Subject: [PATCH 1/2] arrow: emit target ResourceAttrs and map job/instance target labels Adds TargetAttributes and DecodeOptions::target_attributes so ArrowDecoder emits an OTAP ResourceAttrs (ArrowPayloadType = 1) batch attached to resource id 0 on every non-empty decode. Maps Prometheus job and instance labels to OpenTelemetry service.name and service.instance.id alongside extra non-reserved target labels, lets explicit resource attributes override mapped labels, and caches the encoded RecordBatch in TargetAttributes across streaming chunks and repeated scrapes. Towards #35. Signed-off-by: Bartek Plotka --- crates/expfmt/benches/decode.rs | 35 ++- crates/expfmt/src/arrow.rs | 29 ++- crates/expfmt/src/arrow/columns.rs | 56 ++++- crates/expfmt/src/arrow/options.rs | 245 +++++++++++++++++++- crates/expfmt/src/arrow/records.rs | 15 ++ crates/expfmt/src/arrow/schema.rs | 33 +++ crates/expfmt/src/arrow/tests.rs | 207 ++++++++++++++++- crates/scrape-manager/src/scrape/config.rs | 2 +- crates/scrape-manager/src/scrape/manager.rs | 1 + 9 files changed, 585 insertions(+), 38 deletions(-) diff --git a/crates/expfmt/benches/decode.rs b/crates/expfmt/benches/decode.rs index 0d1609a..1c8e4f5 100644 --- a/crates/expfmt/benches/decode.rs +++ b/crates/expfmt/benches/decode.rs @@ -110,7 +110,7 @@ fn main() { let full_bytes = full.as_bytes(); let opts = DecodeOptions::new().with_timestamp_ms(0); - let stream_256_opts = opts.with_max_data_points_per_batch(256); + let stream_256_opts = opts.clone().with_max_data_points_per_batch(256); let budget = DecodeBudget::default(); let mut suite = Suite::from_env("prometheus-expfmt").with_alloc(&ALLOC); @@ -121,19 +121,28 @@ fn main() { ("alltypes.237mfs", full_bytes), ] { let bytes = payload.len() as u64; + let opts_decode = opts.clone(); + let opts_coop = opts.clone(); + let opts_stream_16k = opts.clone(); + let opts_stream_256 = stream_256_opts.clone(); + let opts_stream_256_coop = stream_256_opts.clone(); suite = suite .bench(format!("Decode/case={case}/mode=decode"), move |b| { // Warm internal staging and interner capacities once so B/op and allocs/op // measure steady-state scrape reuse rather than cold capacity growth divided by N. let mut decoder = ArrowDecoder::new(); - black_box(decoder.decode(payload, &Format::TEXT, &opts).unwrap()); + black_box( + decoder + .decode(payload, &Format::TEXT, &opts_decode) + .unwrap(), + ); b.set_bytes(bytes); b.report_allocs(); while b.keep_running() { let records = decoder - .decode(black_box(payload), &Format::TEXT, &opts) + .decode(black_box(payload), &Format::TEXT, &opts_decode) .unwrap(); black_box(records); } @@ -142,7 +151,7 @@ fn main() { format!("Decode/case={case}/mode=decode_cooperative"), move |b| { let mut decoder = ArrowDecoder::new(); - black_box(decoder.decode(payload, &Format::TEXT, &opts).unwrap()); + black_box(decoder.decode(payload, &Format::TEXT, &opts_coop).unwrap()); b.set_bytes(bytes); b.report_allocs(); @@ -150,7 +159,7 @@ fn main() { let records = poll_cooperative(decoder.decode_cooperative( black_box(payload), &Format::TEXT, - &opts, + &opts_coop, budget, || YieldOnce(false), )) @@ -161,21 +170,23 @@ fn main() { ) .bench(format!("Decode/case={case}/mode=stream_16k"), move |b| { let mut decoder = ArrowDecoder::new(); - for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts) { + for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts_stream_16k) { black_box(chunk.unwrap()); } b.set_bytes(bytes); b.report_allocs(); while b.keep_running() { - for chunk in decoder.decode_stream(black_box(payload), &Format::TEXT, &opts) { + for chunk in + decoder.decode_stream(black_box(payload), &Format::TEXT, &opts_stream_16k) + { black_box(chunk.unwrap()); } } }) .bench(format!("Decode/case={case}/mode=stream_256"), move |b| { let mut decoder = ArrowDecoder::new(); - for chunk in decoder.decode_stream(payload, &Format::TEXT, &stream_256_opts) { + for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts_stream_256) { black_box(chunk.unwrap()); } @@ -183,7 +194,7 @@ fn main() { b.report_allocs(); while b.keep_running() { for chunk in - decoder.decode_stream(black_box(payload), &Format::TEXT, &stream_256_opts) + decoder.decode_stream(black_box(payload), &Format::TEXT, &opts_stream_256) { black_box(chunk.unwrap()); } @@ -193,7 +204,9 @@ fn main() { format!("Decode/case={case}/mode=stream_256_cooperative"), move |b| { let mut decoder = ArrowDecoder::new(); - for chunk in decoder.decode_stream(payload, &Format::TEXT, &stream_256_opts) { + for chunk in + decoder.decode_stream(payload, &Format::TEXT, &opts_stream_256_coop) + { black_box(chunk.unwrap()); } @@ -203,7 +216,7 @@ fn main() { let mut stream = decoder.decode_stream( black_box(payload), &Format::TEXT, - &stream_256_opts, + &opts_stream_256_coop, ); loop { match stream.step(budget).unwrap() { diff --git a/crates/expfmt/src/arrow.rs b/crates/expfmt/src/arrow.rs index 046f020..25b8630 100644 --- a/crates/expfmt/src/arrow.rs +++ b/crates/expfmt/src/arrow.rs @@ -49,11 +49,11 @@ use self::intern::StringInterner; pub use self::options::{ CounterSuffixPolicy, DECODE_BUDGET_DURATION_CHECK_INTERVAL, DEFAULT_DECODE_BUDGET_MAX_BYTES, DEFAULT_DECODE_BUDGET_MAX_DURATION, DEFAULT_DECODE_BUDGET_MAX_ENTRIES, - DEFAULT_MAX_DATA_POINTS_PER_BATCH, DecodeBudget, DecodeError, DecodeOptions, UnitSuffixPolicy, - UntypedPolicy, + DEFAULT_MAX_DATA_POINTS_PER_BATCH, DecodeBudget, DecodeError, DecodeOptions, TargetAttributes, + UnitSuffixPolicy, UntypedPolicy, }; pub use self::records::{MetricPayloadType, OtapMetricRecords}; -pub use self::schema::PROMETHEUS_TYPE_METADATA_KEY; +pub use self::schema::{PROMETHEUS_TYPE_METADATA_KEY, SERVICE_INSTANCE_ID_ATTR, SERVICE_NAME_ATTR}; /// Maximum distinct `UnivariateMetrics` rows representable in a single OTAP batch (`u16` IDs). const MAX_METRICS_PER_BATCH: usize = (u16::MAX as usize) + 1; @@ -654,7 +654,7 @@ impl ArrowDecoder { input, cursor: DecodeCursor { parser_state: ActiveParserState::Invalid, - opts: *opts, + opts: opts.clone(), default_ts_ns: None, max_data_points, scrape_id, @@ -686,7 +686,7 @@ impl ArrowDecoder { let (parser, default_ts_ns) = self.init_scrape_parser(b"", format, opts)?; Ok(DecodeCursor { parser_state: parser.into_state(), - opts: *opts, + opts: opts.clone(), default_ts_ns, max_data_points, scrape_id: self.scrape_id, @@ -844,7 +844,7 @@ impl ArrowDecoder { if self.metrics.is_empty() { return Ok(DecodeStep::Done); } - let batch = self.finish_batches(default_ts_ns.is_some())?; + let batch = self.finish_batches(opts, default_ts_ns.is_some())?; if CHUNKED { self.reset_batch(); } @@ -892,7 +892,7 @@ impl ArrowDecoder { self.current_family .start_new(raw_name, &self.series_name_buf); *pending_series = true; - let batch = self.finish_batches(default_ts_ns.is_some())?; + let batch = self.finish_batches(opts, default_ts_ns.is_some())?; self.reset_batch(); return Ok(DecodeStep::Batch(batch)); } @@ -915,7 +915,7 @@ impl ArrowDecoder { self.flush_pending_family(opts, default_ts_ns)?; } *pending_series = true; - let batch = self.finish_batches(default_ts_ns.is_some())?; + let batch = self.finish_batches(opts, default_ts_ns.is_some())?; self.reset_batch(); return Ok(DecodeStep::Batch(batch)); } @@ -927,7 +927,7 @@ impl ArrowDecoder { && self.assembly.is_empty() && self.total_data_points() >= max_data_points { - let batch = self.finish_batches(default_ts_ns.is_some())?; + let batch = self.finish_batches(opts, default_ts_ns.is_some())?; self.reset_batch(); return Ok(DecodeStep::Batch(batch)); } @@ -1549,11 +1549,19 @@ impl ArrowDecoder { Ok(()) } - fn finish_batches(&mut self, has_default_ts: bool) -> Result { + fn finish_batches( + &mut self, + opts: &DecodeOptions, + has_default_ts: bool, + ) -> Result { if self.metrics.is_empty() { return Ok(OtapMetricRecords::default()); } + let resource_attrs = match opts.target_attributes.as_deref() { + Some(target) => target.resource_attrs_batch()?, + None => None, + }; let univariate_metrics = Some(self.metrics.build_univariate_metrics(&mut self.scratch)?); let number_data_points = self.num_dp.build_batch(has_default_ts, &mut self.scratch)?; let summary_data_points = self.sum_dp.build_batch(has_default_ts, &mut self.scratch)?; @@ -1589,6 +1597,7 @@ impl ArrowDecoder { let metric_attrs = Some(self.metrics.build_metric_attrs(&mut self.scratch)?); Ok(OtapMetricRecords { + resource_attrs, univariate_metrics, metric_attrs, number_data_points, diff --git a/crates/expfmt/src/arrow/columns.rs b/crates/expfmt/src/arrow/columns.rs index b5dd5fd..360e490 100644 --- a/crates/expfmt/src/arrow/columns.rs +++ b/crates/expfmt/src/arrow/columns.rs @@ -13,6 +13,7 @@ //! Reusable zero-allocation column and table staging buffers for [`super::ArrowDecoder`]. +use std::collections::BTreeMap; use std::sync::Arc; use arrow_array::types::{UInt8Type, UInt16Type}; @@ -30,8 +31,9 @@ use super::schema::{ OTAP_METRIC_TYPE_HISTOGRAM, OTAP_METRIC_TYPE_SUM, UTC_TIME_ZONE, ValEncoding, dp_attrs_schema, exemplars_schema, histogram_bucket_counts_item_field, histogram_dp_schema, histogram_explicit_bounds_item_field, metric_attrs_key_dict, metric_attrs_schema, - metric_attrs_type_val_dict, number_dp_schema, resource_struct_fields, summary_dp_schema, - summary_quantile_item_field, summary_quantile_struct_fields, univariate_metrics_schema, + metric_attrs_type_val_dict, number_dp_schema, resource_attrs_schema, resource_struct_fields, + summary_dp_schema, summary_quantile_item_field, summary_quantile_struct_fields, + univariate_metrics_schema, }; /// Reusable scratch vectors used when materializing Arrow columns without reallocating. @@ -748,6 +750,56 @@ impl AttrTable { } } +/// Builds an OTAP `ResourceAttrs` (`attributes_16`) [`RecordBatch`] attached to `parent_id = 0` +/// (matching `UnivariateMetrics.resource.id = 0`). +/// +/// Returns `Ok(None)` when `attrs` is empty so empty tables are omitted per OTAP invariants. +pub(super) fn build_resource_attrs_batch( + attrs: &BTreeMap, +) -> Result, DecodeError> { + if attrs.is_empty() { + return Ok(None); + } + + let num_attrs = attrs.len(); + let key_bytes: usize = attrs.keys().map(String::len).sum(); + let val_bytes: usize = attrs.values().map(String::len).sum(); + let mut key_interner = StringInterner::with_capacity(num_attrs, key_bytes); + let mut val_interner = StringInterner::with_capacity(num_attrs, val_bytes); + let mut pairs = Vec::with_capacity(num_attrs); + + for (k, v) in attrs { + let kid = + key_interner + .intern(k.as_bytes()) + .ok_or_else(|| DecodeError::CapacityOverflow { + table: "ResourceAttrs.key", + count: key_interner.len(), + })?; + let vid = + val_interner + .intern(v.as_bytes()) + .ok_or_else(|| DecodeError::CapacityOverflow { + table: "ResourceAttrs.str", + count: val_interner.len(), + })?; + pairs.push((kid, vid)); + } + + let enc = AttrEncoding::new(&key_interner, &val_interner, true); + let mut scratch = ScratchBuffers::with_capacity(0, 0, num_attrs); + scratch.u16_buf.clear(); + scratch.u16_buf.resize(num_attrs, 0); + let parent_id_arr: ArrayRef = + Arc::new(UInt16Array::new(scalar_from_slice(&scratch.u16_buf), None)); + let (key_arr, type_arr, val_arr) = scratch.build_attr_columns(&pairs, &enc, "ResourceAttrs")?; + + Ok(Some(RecordBatch::try_new( + resource_attrs_schema(enc.key_enc, enc.val_enc), + vec![parent_id_arr, key_arr, type_arr, val_arr], + )?)) +} + /// Staging buffer for an optional fixed-size binary column (`span_id` or `trace_id` on exemplars). #[derive(Default)] pub(super) struct SparseFixedBinaryCol { diff --git a/crates/expfmt/src/arrow/options.rs b/crates/expfmt/src/arrow/options.rs index 2b66367..2ff2692 100644 --- a/crates/expfmt/src/arrow/options.rs +++ b/crates/expfmt/src/arrow/options.rs @@ -14,12 +14,18 @@ //! Configuration policies, [`DecodeOptions`], [`DecodeBudget`], and [`DecodeError`] for //! [`super::ArrowDecoder`]. +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; use std::time::Duration; +use arrow_array::RecordBatch; use arrow_schema::ArrowError; use crate::parse::ParseError; +use super::columns::build_resource_attrs_batch; +use super::schema::{SERVICE_INSTANCE_ID_ATTR, SERVICE_NAME_ATTR}; + /// Default upper bound on data points emitted per [`super::OtapMetricRecords`] chunk in /// [`super::ArrowDecoder::decode_stream`]. pub const DEFAULT_MAX_DATA_POINTS_PER_BATCH: usize = 16_384; @@ -148,8 +154,217 @@ pub enum UntypedPolicy { Drop, } +/// Reserved Prometheus label prefix excluded from public target labels and resource attributes. +const RESERVED_LABEL_PREFIX: &str = "__"; +const JOB_LABEL: &str = "job"; +const INSTANCE_LABEL: &str = "instance"; + +/// Target-level Prometheus labels and OpenTelemetry resource attributes attached once per scrape +/// to the OTAP `ResourceAttrs` (`ArrowPayloadType = 1`) table. +/// +/// Non-reserved, non-empty Prometheus target labels map into OpenTelemetry resource attributes: +/// - `job` maps to [`SERVICE_NAME_ATTR`] (`"service.name"`). +/// - `instance` maps to [`SERVICE_INSTANCE_ID_ATTR`] (`"service.instance.id"`). +/// - Any other public target label attaches verbatim as a resource attribute. +/// - Internal labels starting with `"__"` and empty target label values are ignored. +/// +/// Explicit resource attributes added via [`Self::with_resource_attribute`] or +/// [`Self::with_resource_attributes`] take precedence over mapped target labels. The encoded +/// `ResourceAttrs` [`RecordBatch`] is materialized lazily on first decode and shared across +/// streaming chunks and repeated scrapes via `Arc`. +#[derive(Debug, Default)] +pub struct TargetAttributes { + target_labels: BTreeMap, + explicit_resource_attrs: BTreeMap, + resolved_resource_attrs: BTreeMap, + cached_batch: OnceLock>, +} + +impl Clone for TargetAttributes { + fn clone(&self) -> Self { + let cached_batch = match self.cached_batch.get() { + Some(batch) => OnceLock::from(batch.clone()), + None => OnceLock::new(), + }; + Self { + target_labels: self.target_labels.clone(), + explicit_resource_attrs: self.explicit_resource_attrs.clone(), + resolved_resource_attrs: self.resolved_resource_attrs.clone(), + cached_batch, + } + } +} + +impl PartialEq for TargetAttributes { + fn eq(&self, other: &Self) -> bool { + self.target_labels == other.target_labels + && self.explicit_resource_attrs == other.explicit_resource_attrs + && self.resolved_resource_attrs == other.resolved_resource_attrs + } +} + +impl Eq for TargetAttributes {} + +impl TargetAttributes { + /// Creates an empty [`TargetAttributes`] set. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Creates a [`TargetAttributes`] set from Prometheus target labels, ignoring reserved `"__*"` + /// labels and empty values. + #[must_use] + pub fn from_target_labels(labels: I) -> Self + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + Self::new().with_target_labels(labels) + } + + /// Sets the Prometheus `job` target label, mapped to `service.name` in `ResourceAttrs`. + #[must_use] + pub fn with_job(self, job: impl Into) -> Self { + self.with_target_label(JOB_LABEL, job) + } + + /// Sets the Prometheus `instance` target label, mapped to `service.instance.id` in + /// `ResourceAttrs`. + #[must_use] + pub fn with_instance(self, instance: impl Into) -> Self { + self.with_target_label(INSTANCE_LABEL, instance) + } + + /// Inserts or updates a single Prometheus target label. + /// + /// Reserved `"__*"` label names are ignored. Setting an empty value removes any previous value + /// for `key`, matching Prometheus label semantics. + #[must_use] + pub fn with_target_label(mut self, key: impl Into, value: impl Into) -> Self { + self.insert_target_label_raw(key.into(), value.into()); + self.rebuild_resolved(); + self + } + + /// Inserts or updates multiple Prometheus target labels. + #[must_use] + pub fn with_target_labels(mut self, labels: I) -> Self + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + for (k, v) in labels { + self.insert_target_label_raw(k.as_ref().to_owned(), v.as_ref().to_owned()); + } + self.rebuild_resolved(); + self + } + + /// Inserts or updates an explicit OpenTelemetry resource attribute. + /// + /// Explicit resource attributes take precedence over attributes mapped from Prometheus target + /// labels (such as `job` -> `service.name` and `instance` -> `service.instance.id`). + #[must_use] + pub fn with_resource_attribute( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + let key = key.into(); + if !key.is_empty() { + self.explicit_resource_attrs.insert(key, value.into()); + self.rebuild_resolved(); + } + self + } + + /// Inserts or updates multiple explicit OpenTelemetry resource attributes. + #[must_use] + pub fn with_resource_attributes(mut self, attrs: I) -> Self + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + for (k, v) in attrs { + let key = k.as_ref(); + if !key.is_empty() { + self.explicit_resource_attrs + .insert(key.to_owned(), v.as_ref().to_owned()); + } + } + self.rebuild_resolved(); + self + } + + /// Returns the public (non-reserved, non-empty) Prometheus target labels. + #[must_use] + pub fn target_labels(&self) -> &BTreeMap { + &self.target_labels + } + + /// Returns the resolved OpenTelemetry resource attributes emitted in `ResourceAttrs`. + #[must_use] + pub fn resource_attributes(&self) -> &BTreeMap { + &self.resolved_resource_attrs + } + + /// Returns `true` when neither Prometheus target labels nor explicit resource attributes are + /// configured. + #[must_use] + pub fn is_empty(&self) -> bool { + self.target_labels.is_empty() && self.resolved_resource_attrs.is_empty() + } + + pub(super) fn resource_attrs_batch(&self) -> Result, DecodeError> { + if let Some(batch) = self.cached_batch.get() { + return Ok(batch.clone()); + } + let built = build_resource_attrs_batch(&self.resolved_resource_attrs)?; + let cached = self.cached_batch.get_or_init(|| built); + Ok(cached.clone()) + } + + fn insert_target_label_raw(&mut self, key: String, value: String) { + if key.is_empty() || key.starts_with(RESERVED_LABEL_PREFIX) { + return; + } + if value.is_empty() { + self.target_labels.remove(&key); + } else { + self.target_labels.insert(key, value); + } + } + + fn rebuild_resolved(&mut self) { + self.resolved_resource_attrs.clear(); + for (k, v) in &self.target_labels { + if k != JOB_LABEL && k != INSTANCE_LABEL { + self.resolved_resource_attrs.insert(k.clone(), v.clone()); + } + } + if let Some(job) = self.target_labels.get(JOB_LABEL) { + self.resolved_resource_attrs + .entry(SERVICE_NAME_ATTR.to_owned()) + .or_insert_with(|| job.clone()); + } + if let Some(instance) = self.target_labels.get(INSTANCE_LABEL) { + self.resolved_resource_attrs + .entry(SERVICE_INSTANCE_ID_ATTR.to_owned()) + .or_insert_with(|| instance.clone()); + } + for (k, v) in &self.explicit_resource_attrs { + self.resolved_resource_attrs.insert(k.clone(), v.clone()); + } + self.cached_batch = OnceLock::new(); + } +} + /// Options controlling [`super::ArrowDecoder::decode`] and [`super::ArrowDecoder::decode_stream`]. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct DecodeOptions { /// Default timestamp in milliseconds since the Unix epoch applied to samples that omit an /// explicit wire timestamp. When `None`, untimestamped samples leave `time_unix_nano` unset. @@ -164,6 +379,9 @@ pub struct DecodeOptions { pub unit_suffix_policy: UnitSuffixPolicy, /// Projection policy for untyped (`unknown`) metric families. pub untyped_policy: UntypedPolicy, + /// Optional target labels and OpenTelemetry resource attributes attached to every non-empty + /// decoded batch as `ResourceAttrs` (`ArrowPayloadType = 1`). + pub target_attributes: Option>, } impl Default for DecodeOptions { @@ -174,8 +392,8 @@ impl Default for DecodeOptions { impl DecodeOptions { /// Creates default [`DecodeOptions`] with no fallback timestamp, - /// [`DEFAULT_MAX_DATA_POINTS_PER_BATCH`] data points per streamed batch, and default - /// translation policies. + /// [`DEFAULT_MAX_DATA_POINTS_PER_BATCH`] data points per streamed batch, default translation + /// policies, and no target resource attributes. #[must_use] pub const fn new() -> Self { Self { @@ -184,12 +402,13 @@ impl DecodeOptions { counter_suffix_policy: CounterSuffixPolicy::Keep, unit_suffix_policy: UnitSuffixPolicy::Keep, untyped_policy: UntypedPolicy::Gauge, + target_attributes: None, } } /// Sets the fallback timestamp in milliseconds for samples without an explicit wire timestamp. #[must_use] - pub const fn with_timestamp_ms(mut self, timestamp_ms: i64) -> Self { + pub fn with_timestamp_ms(mut self, timestamp_ms: i64) -> Self { self.timestamp_ms = Some(timestamp_ms); self } @@ -197,34 +416,38 @@ impl DecodeOptions { /// Sets the maximum number of data points materialized per batch in /// [`super::ArrowDecoder::decode_stream`]. #[must_use] - pub const fn with_max_data_points_per_batch( - mut self, - max_data_points_per_batch: usize, - ) -> Self { + pub fn with_max_data_points_per_batch(mut self, max_data_points_per_batch: usize) -> Self { self.max_data_points_per_batch = max_data_points_per_batch; self } /// Sets the [`CounterSuffixPolicy`] for counter metric names. #[must_use] - pub const fn with_counter_suffix_policy(mut self, policy: CounterSuffixPolicy) -> Self { + pub fn with_counter_suffix_policy(mut self, policy: CounterSuffixPolicy) -> Self { self.counter_suffix_policy = policy; self } /// Sets the [`UnitSuffixPolicy`] for `# UNIT` suffixes on metric names. #[must_use] - pub const fn with_unit_suffix_policy(mut self, policy: UnitSuffixPolicy) -> Self { + pub fn with_unit_suffix_policy(mut self, policy: UnitSuffixPolicy) -> Self { self.unit_suffix_policy = policy; self } /// Sets the [`UntypedPolicy`] for untyped (`unknown`) metric families. #[must_use] - pub const fn with_untyped_policy(mut self, policy: UntypedPolicy) -> Self { + pub fn with_untyped_policy(mut self, policy: UntypedPolicy) -> Self { self.untyped_policy = policy; self } + + /// Sets target labels and OpenTelemetry resource attributes emitted in `ResourceAttrs`. + #[must_use] + pub fn with_target_attributes(mut self, target_attributes: Arc) -> Self { + self.target_attributes = Some(target_attributes); + self + } } /// Errors returned by [`super::ArrowDecoder::decode`]. diff --git a/crates/expfmt/src/arrow/records.rs b/crates/expfmt/src/arrow/records.rs index fd7312f..568ef79 100644 --- a/crates/expfmt/src/arrow/records.rs +++ b/crates/expfmt/src/arrow/records.rs @@ -35,6 +35,8 @@ use arrow_array::RecordBatch; #[repr(i32)] #[non_exhaustive] pub enum MetricPayloadType { + /// Target-level resource attributes (`ArrowPayloadType::ResourceAttrs = 1`). + ResourceAttrs = 1, /// Metric descriptor table (`ArrowPayloadType::UnivariateMetrics = 10`). UnivariateMetrics = 10, /// Scalar gauge, counter, and untyped data points (`ArrowPayloadType::NumberDataPoints = 11`). @@ -83,6 +85,7 @@ impl MetricPayloadType { /// Tables with zero rows are omitted (`None`) to match OTAP batch store invariants. #[derive(Clone, Debug, Default, PartialEq)] pub struct OtapMetricRecords { + pub(super) resource_attrs: Option, pub(super) univariate_metrics: Option, pub(super) metric_attrs: Option, pub(super) number_data_points: Option, @@ -98,6 +101,12 @@ pub struct OtapMetricRecords { } impl OtapMetricRecords { + /// Returns the `ResourceAttrs` table (`ArrowPayloadType = 1`), if non-empty. + #[must_use] + pub fn resource_attrs(&self) -> Option<&RecordBatch> { + self.resource_attrs.as_ref() + } + /// Returns the `UnivariateMetrics` table (`ArrowPayloadType = 10`), if non-empty. #[must_use] pub fn univariate_metrics(&self) -> Option<&RecordBatch> { @@ -174,6 +183,7 @@ impl OtapMetricRecords { #[must_use] pub fn get(&self, payload_type: MetricPayloadType) -> Option<&RecordBatch> { match payload_type { + MetricPayloadType::ResourceAttrs => self.resource_attrs.as_ref(), MetricPayloadType::UnivariateMetrics => self.univariate_metrics.as_ref(), MetricPayloadType::NumberDataPoints => self.number_data_points.as_ref(), MetricPayloadType::SummaryDataPoints => self.summary_data_points.as_ref(), @@ -194,6 +204,10 @@ impl OtapMetricRecords { /// Iterates over all non-empty `(MetricPayloadType, &RecordBatch)` pairs in wire order. pub fn iter(&self) -> impl Iterator { [ + ( + MetricPayloadType::ResourceAttrs, + self.resource_attrs.as_ref(), + ), ( MetricPayloadType::UnivariateMetrics, self.univariate_metrics.as_ref(), @@ -247,6 +261,7 @@ impl OtapMetricRecords { /// Consumes `self` and yields all non-empty `(MetricPayloadType, RecordBatch)` pairs. pub fn into_batches(self) -> impl Iterator { [ + (MetricPayloadType::ResourceAttrs, self.resource_attrs), ( MetricPayloadType::UnivariateMetrics, self.univariate_metrics, diff --git a/crates/expfmt/src/arrow/schema.rs b/crates/expfmt/src/arrow/schema.rs index 23b6804..177a776 100644 --- a/crates/expfmt/src/arrow/schema.rs +++ b/crates/expfmt/src/arrow/schema.rs @@ -49,6 +49,12 @@ pub(super) const ATTR_VALUE_TYPE_STR: u8 = 1; /// Metric-level metadata attribute key recording the original Prometheus exposition type. pub const PROMETHEUS_TYPE_METADATA_KEY: &str = "prometheus.type"; +/// OpenTelemetry resource attribute key mapped from the Prometheus `job` target label. +pub const SERVICE_NAME_ATTR: &str = "service.name"; + +/// OpenTelemetry resource attribute key mapped from the Prometheus `instance` target label. +pub const SERVICE_INSTANCE_ID_ATTR: &str = "service.instance.id"; + /// UTC timezone tag required by section 5.5.2 of the OTAP specification on `Timestamp(Nanosecond)`. pub(super) const UTC_TIME_ZONE: &str = "UTC"; @@ -518,6 +524,33 @@ static DP_ATTRS_SCHEMAS: LazyLock<[SchemaRef; 6]> = LazyLock::new(|| { ] }); +fn build_u16_attrs_schema(key_enc: KeyEncoding, val_enc: ValEncoding) -> SchemaRef { + let (key_dt, val_dt) = attr_key_val_data_types(key_enc, val_enc); + Arc::new(Schema::new(vec![ + Field::new("parent_id", DataType::UInt16, false).with_metadata(plain_id_metadata()), + Field::new("key", key_dt, false), + Field::new("type", DataType::UInt8, false), + Field::new("str", val_dt, true), + ])) +} + +static RESOURCE_ATTRS_SCHEMAS: LazyLock<[SchemaRef; 6]> = LazyLock::new(|| { + [ + build_u16_attrs_schema(KeyEncoding::DictU8, ValEncoding::DictU16), + build_u16_attrs_schema(KeyEncoding::DictU8, ValEncoding::Utf8), + build_u16_attrs_schema(KeyEncoding::DictU16, ValEncoding::DictU16), + build_u16_attrs_schema(KeyEncoding::DictU16, ValEncoding::Utf8), + build_u16_attrs_schema(KeyEncoding::Utf8, ValEncoding::DictU16), + build_u16_attrs_schema(KeyEncoding::Utf8, ValEncoding::Utf8), + ] +}); + +#[inline] +pub(super) fn resource_attrs_schema(key_enc: KeyEncoding, val_enc: ValEncoding) -> SchemaRef { + let idx = (key_enc as usize) * 2 + (val_enc as usize); + Arc::clone(&RESOURCE_ATTRS_SCHEMAS[idx]) +} + #[inline] pub(super) fn dp_attrs_schema(key_enc: KeyEncoding, val_enc: ValEncoding) -> SchemaRef { let idx = (key_enc as usize) * 2 + (val_enc as usize); diff --git a/crates/expfmt/src/arrow/tests.rs b/crates/expfmt/src/arrow/tests.rs index 7b0b074..a8a6b5c 100644 --- a/crates/expfmt/src/arrow/tests.rs +++ b/crates/expfmt/src/arrow/tests.rs @@ -809,7 +809,7 @@ fn lifetime_free_decode_cursor_supports_owned_job_across_steps() { ); // 2. Chunked `start_decode_stream` driven 1 entry at a time. - let stream_opts = opts.with_max_data_points_per_batch(1); + let stream_opts = opts.clone().with_max_data_points_per_batch(1); let expected_chunks: Vec<_> = ref_dec .decode_stream( om_payload.as_bytes(), @@ -919,7 +919,7 @@ fn lifetime_free_decode_cursor_supports_owned_job_across_steps() { )); assert!(matches!( job.decoder - .start_decode_stream(&Format::TEXT, &opts.with_timestamp_ms(i64::MAX)), + .start_decode_stream(&Format::TEXT, &opts.clone().with_timestamp_ms(i64::MAX)), Err(DecodeError::TimestampOverflow { .. }) )); @@ -1117,7 +1117,7 @@ fn cooperative_async_helpers_respect_fast_path_and_yield_budgets() { assert_eq!(coop_polls, coop_yields); // 3. `DecodeStream::next_cooperative` yields between steps while streaming chunks. - let stream_opts = opts.with_max_data_points_per_batch(15); + let stream_opts = opts.clone().with_max_data_points_per_batch(15); let expected_chunks: Vec<_> = dec .decode_stream(bytes, &Format::TEXT, &stream_opts) .collect::, _>>() @@ -1194,3 +1194,204 @@ fn cooperative_async_helpers_respect_fast_path_and_yield_budgets() { assert!(stream_err_res.1.is_none()); assert_eq!(stream_err_polls, 1); } + +#[test] +fn target_attributes_emit_resource_attrs_and_map_job_and_instance() { + use std::collections::BTreeMap; + use std::sync::Arc; + + use arrow_array::UInt16Array; + + let target = Arc::new( + TargetAttributes::from_target_labels([ + ("__address__", "10.0.0.1:9100"), + ("__meta_kubernetes_pod_name", "api-0"), + ("job", "api-server"), + ("instance", "10.0.0.1:9100"), + ("env", "prod"), + ("empty_label", ""), + ]) + .with_resource_attribute("cloud.region", "us-central1"), + ); + + assert_eq!( + target.target_labels(), + &BTreeMap::from([ + ("env".to_owned(), "prod".to_owned()), + ("instance".to_owned(), "10.0.0.1:9100".to_owned()), + ("job".to_owned(), "api-server".to_owned()), + ]) + ); + assert_eq!( + target.resource_attributes(), + &BTreeMap::from([ + ("cloud.region".to_owned(), "us-central1".to_owned()), + ("env".to_owned(), "prod".to_owned()), + ( + SERVICE_INSTANCE_ID_ATTR.to_owned(), + "10.0.0.1:9100".to_owned() + ), + (SERVICE_NAME_ATTR.to_owned(), "api-server".to_owned()), + ]) + ); + + let opts = DecodeOptions::new() + .with_timestamp_ms(1_700_000_000_000) + .with_target_attributes(Arc::clone(&target)); + + let payload = + b"# TYPE reqs_total counter\nreqs_total{method=\"GET\"} 1\nreqs_total{method=\"POST\"} 2\n"; + let mut dec = ArrowDecoder::new(); + let records = dec.decode(payload, &Format::TEXT, &opts).unwrap(); + + let res_batch = records + .resource_attrs() + .expect("ResourceAttrs emitted on non-empty scrape"); + assert_eq!( + records.get(MetricPayloadType::ResourceAttrs), + Some(res_batch) + ); + assert_eq!(res_batch.num_rows(), 4); + assert_eq!( + res_batch + .schema() + .field_with_name("parent_id") + .unwrap() + .metadata() + .get("encoding") + .map(String::as_str), + Some("plain") + ); + + let parent_ids = res_batch + .column_by_name("parent_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let keys = res_batch + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::>() + .unwrap(); + let key_vals = keys + .values() + .as_any() + .downcast_ref::() + .unwrap(); + let types = res_batch + .column_by_name("type") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let vals = res_batch + .column_by_name("str") + .unwrap() + .as_any() + .downcast_ref::>() + .unwrap(); + let str_vals = vals + .values() + .as_any() + .downcast_ref::() + .unwrap(); + + let mut extracted = BTreeMap::new(); + for row in 0..res_batch.num_rows() { + assert_eq!(parent_ids.value(row), 0); + assert_eq!(types.value(row), 1); + let k = key_vals.value(usize::from(keys.keys().value(row))); + let v = str_vals.value(usize::from(vals.keys().value(row))); + extracted.insert(k.to_owned(), v.to_owned()); + } + assert_eq!(&extracted, target.resource_attributes()); + + // Multi-chunk streaming attaches `ResourceAttrs` to every emitted chunk. + let stream_opts = opts.clone().with_max_data_points_per_batch(1); + let chunks: Vec<_> = dec + .decode_stream(payload, &Format::TEXT, &stream_opts) + .collect::, _>>() + .unwrap(); + assert_eq!(chunks.len(), 2); + for chunk in &chunks { + assert_eq!(chunk.resource_attrs(), Some(res_batch)); + } + + // Empty and all-dropped scrapes omit `ResourceAttrs`. + let empty = dec.decode(b"", &Format::TEXT, &opts).unwrap(); + assert!(empty.is_empty()); + assert!(empty.resource_attrs().is_none()); + + let dropped_opts = opts.clone().with_untyped_policy(UntypedPolicy::Drop); + let dropped = dec + .decode(b"untyped_only 1\n", &Format::TEXT, &dropped_opts) + .unwrap(); + assert!(dropped.is_empty()); + assert!(dropped.resource_attrs().is_none()); + + // Explicit resource attributes override `job` -> `service.name` while keeping `job` in + // `target_labels`. + let overridden = TargetAttributes::new() + .with_job("prom-job") + .with_instance("host:9090") + .with_resource_attribute(SERVICE_NAME_ATTR, "explicit-service"); + assert_eq!( + overridden.target_labels().get("job").map(String::as_str), + Some("prom-job") + ); + assert_eq!( + overridden + .resource_attributes() + .get(SERVICE_NAME_ATTR) + .map(String::as_str), + Some("explicit-service") + ); + assert_eq!( + overridden + .resource_attributes() + .get(SERVICE_INSTANCE_ID_ATTR) + .map(String::as_str), + Some("host:9090") + ); + + // Reserved-only and cleared target labels stay empty and omit `ResourceAttrs`. + let reserved_only = TargetAttributes::from_target_labels([ + ("__address__", "10.0.0.1:9100"), + ("__meta_foo", "bar"), + ]) + .with_job("temp") + .with_job(""); + assert!(reserved_only.is_empty()); + let reserved_opts = DecodeOptions::new().with_target_attributes(Arc::new(reserved_only)); + let reserved_records = dec.decode(b"m 1\n", &Format::TEXT, &reserved_opts).unwrap(); + assert!(reserved_records.resource_attrs().is_none()); + + // `Clone` and `Eq` treat cached vs uncached `TargetAttributes` identically. + let uncached = TargetAttributes::new() + .with_job("api-server") + .with_instance("10.0.0.1:9100") + .with_target_label("env", "prod") + .with_resource_attribute("cloud.region", "us-central1"); + assert_eq!(*target, uncached); + assert_eq!(target.as_ref().clone(), uncached); + + // >256 resource attributes widen `ResourceAttrs.key` from UInt8 to UInt16 dictionary. + let mut wide_target = TargetAttributes::new(); + for i in 0..260 { + wide_target = wide_target.with_target_label(format!("res_k_{i}"), "v"); + } + let wide_opts = DecodeOptions::new().with_target_attributes(Arc::new(wide_target)); + let wide_records = dec.decode(b"m 1\n", &Format::TEXT, &wide_opts).unwrap(); + let wide_res = wide_records.resource_attrs().unwrap(); + assert_eq!(wide_res.num_rows(), 260); + assert_eq!( + wide_res + .schema() + .field_with_name("key") + .unwrap() + .data_type(), + &DataType::Dictionary(Box::new(DataType::UInt16), Box::new(DataType::Utf8)) + ); +} diff --git a/crates/scrape-manager/src/scrape/config.rs b/crates/scrape-manager/src/scrape/config.rs index 46f611a..0b93c7f 100644 --- a/crates/scrape-manager/src/scrape/config.rs +++ b/crates/scrape-manager/src/scrape/config.rs @@ -431,7 +431,7 @@ impl ScrapeConfig { /// Sets [`ScrapeConfig::decode_options`]. #[cfg(feature = "arrow")] #[must_use] - pub const fn with_decode_options(mut self, options: DecodeOptions) -> Self { + pub fn with_decode_options(mut self, options: DecodeOptions) -> Self { self.decode_options = options; self } diff --git a/crates/scrape-manager/src/scrape/manager.rs b/crates/scrape-manager/src/scrape/manager.rs index 4c7da88..aec1863 100644 --- a/crates/scrape-manager/src/scrape/manager.rs +++ b/crates/scrape-manager/src/scrape/manager.rs @@ -507,6 +507,7 @@ impl ScrapeManager { let decode_opts = self .config .decode_options + .clone() .with_timestamp_ms(scrape_timestamp_ms) .with_max_data_points_per_batch(chunk_limit); From 16ce7f91807393fee4a6e780f8242bf86aee2f97 Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Wed, 18 Nov 2026 05:05:56 +0000 Subject: [PATCH 2/2] tests/otap-compat: verify ResourceAttrs schema, OTLP round-trip, and bounded memory Checks MetricPayloadType::ResourceAttrs discriminant parity, validates decoder-emitted ResourceAttrs against upstream OTAP schemas, verifies downstream apply_attribute_transform upserts on top of pre-populated ResourceAttrs, and exercises cached TargetAttributes in the dhat bounded-memory loop. Signed-off-by: Bartek Plotka --- tests/otap-compat/tests/arrow_otap.rs | 83 ++++++++++++++++++++- tests/otap-compat/tests/dhat_bounded_mem.rs | 26 +++++-- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/tests/otap-compat/tests/arrow_otap.rs b/tests/otap-compat/tests/arrow_otap.rs index 33f0f29..ec4c8e6 100644 --- a/tests/otap-compat/tests/arrow_otap.rs +++ b/tests/otap-compat/tests/arrow_otap.rs @@ -36,7 +36,7 @@ use otel_arrow_dfe_pdata::testing::round_trip::{otap_to_otlp, to_metrics_data}; use prometheus_expfmt::arrow::{ ArrowDecoder, CounterSuffixPolicy, DecodeBudget, DecodeCursor, DecodeError, DecodeOptions, DecodeStep, MetricPayloadType, OtapMetricRecords, PROMETHEUS_TYPE_METADATA_KEY, - UnitSuffixPolicy, UntypedPolicy, + SERVICE_INSTANCE_ID_ATTR, SERVICE_NAME_ATTR, TargetAttributes, UnitSuffixPolicy, UntypedPolicy, }; use prometheus_expfmt::format::Format; @@ -100,6 +100,10 @@ fn number_dp(attrs: &[(&str, &str)], time_unix_nano: u64, val: f64) -> NumberDat #[test] fn metric_payload_type_discriminants_match_otap_proto() { let cases = [ + ( + MetricPayloadType::ResourceAttrs, + ArrowPayloadType::ResourceAttrs, + ), ( MetricPayloadType::UnivariateMetrics, ArrowPayloadType::UnivariateMetrics, @@ -879,7 +883,7 @@ fn decode_stream_chunks_validate_in_otap_and_handle_more_than_u16_max_metrics() // Stream the full 237-family payload in 64-point chunks and verify semantic equivalence with // single-batch decoding. - let stream_opts = single_opts.with_max_data_points_per_batch(64); + let stream_opts = single_opts.clone().with_max_data_points_per_batch(64); let mut chunk_count = 0usize; let mut streamed_dps = 0usize; let mut streamed_otlp = Vec::new(); @@ -906,7 +910,7 @@ fn decode_stream_chunks_validate_in_otap_and_handle_more_than_u16_max_metrics() .decode(&om_bench, &Format::OPEN_METRICS_1_0_0, &single_opts) .expect("single-batch OpenMetrics decode succeeds"); let om_single_otlp = otap_to_otlp(&to_otap_arrow_records(&om_single)); - let om_stream_opts = single_opts.with_max_data_points_per_batch(16); + let om_stream_opts = single_opts.clone().with_max_data_points_per_batch(16); let mut om_streamed_otlp = Vec::new(); for chunk_res in decoder.decode_stream(&om_bench, &Format::OPEN_METRICS_1_0_0, &om_stream_opts) { @@ -1020,6 +1024,77 @@ fn univariate_metrics_resource_id_enables_otap_resource_attribute_upserts() { assert!(res_attrs.contains(&kv_str("service.instance.id", "10.0.0.1:9001"))); } +#[test] +fn target_attributes_round_trip_through_otap_resource_attrs_and_downstream_upserts() { + use std::collections::BTreeMap; + use std::sync::Arc; + + use otel_arrow_dfe_pdata::otap::transform::{ + AttributesTransform, LiteralValue, UpsertTransform, apply_attribute_transform, + }; + + let target = Arc::new( + TargetAttributes::from_target_labels([ + ("__address__", "10.52.2.15:9001"), + ("job", "avalanche"), + ("instance", "10.52.2.15:9001"), + ("cluster", "us-central1-a"), + ]) + .with_resource_attribute("deployment.environment", "staging"), + ); + + let mut decoder = ArrowDecoder::new(); + let opts = DecodeOptions::new() + .with_timestamp_ms(1_700_000_000_000) + .with_target_attributes(target); + let decoded = decoder + .decode( + b"# TYPE up gauge\nup{series_id=\"0\"} 1\n# TYPE req_total counter\nreq_total 5\n", + &Format::TEXT, + &opts, + ) + .expect("decode succeeds"); + assert!(decoded.resource_attrs().is_some()); + + let mut otap = to_otap_arrow_records(&decoded); + assert!(otap.get(ArrowPayloadType::ResourceAttrs).is_some()); + + let downstream = AttributesTransform { + upsert: Some(UpsertTransform::new(BTreeMap::from([( + "gcp.project_id".into(), + LiteralValue::Str("gpe-test-1".into()), + )]))), + ..Default::default() + }; + apply_attribute_transform( + &mut otap, + ArrowPayloadType::ResourceAttrs, + &downstream, + false, + ) + .expect("downstream resource attribute upsert succeeds on pre-populated ResourceAttrs"); + + otap.encode_transport_optimized() + .expect("transport optimize succeeds"); + otap.decode_transport_optimized_ids() + .expect("transport decode IDs succeeds"); + + let OtlpProtoMessage::Metrics(otlp) = otap_to_otlp(&otap) else { + panic!("expected OTLP MetricsData"); + }; + assert_eq!(otlp.resource_metrics.len(), 1); + let res_attrs = &otlp.resource_metrics[0] + .resource + .as_ref() + .expect("resource present") + .attributes; + assert!(res_attrs.contains(&kv_str(SERVICE_NAME_ATTR, "avalanche"))); + assert!(res_attrs.contains(&kv_str(SERVICE_INSTANCE_ID_ATTR, "10.52.2.15:9001"))); + assert!(res_attrs.contains(&kv_str("cluster", "us-central1-a"))); + assert!(res_attrs.contains(&kv_str("deployment.environment", "staging"))); + assert!(res_attrs.contains(&kv_str("gcp.project_id", "gpe-test-1"))); +} + async fn probe_scheduling_lateness( mut completed: tokio::sync::oneshot::Receiver, period: Duration, @@ -1140,7 +1215,7 @@ fn cooperative_decode_bounds_local_set_timer_scheduling_lateness_and_validates_i last_records = Some(records); } - let stream_opts = opts.with_max_data_points_per_batch(64); + let stream_opts = opts.clone().with_max_data_points_per_batch(64); let stream_budget = budget.with_max_entries(32); let mut stream_yields = 0usize; let mut streamed_otlp = Vec::new(); diff --git a/tests/otap-compat/tests/dhat_bounded_mem.rs b/tests/otap-compat/tests/dhat_bounded_mem.rs index 3ca7d15..a62eaa3 100644 --- a/tests/otap-compat/tests/dhat_bounded_mem.rs +++ b/tests/otap-compat/tests/dhat_bounded_mem.rs @@ -18,11 +18,12 @@ use std::cell::RefCell; use std::fs; use std::path::Path; use std::rc::Rc; +use std::sync::Arc; use otel_arrow_dfe_pdata::otap::{Metrics, OtapArrowRecords, OtapBatchStore}; use otel_arrow_dfe_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; use prometheus_expfmt::arrow::{ - ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep, OtapMetricRecords, + ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep, OtapMetricRecords, TargetAttributes, }; use prometheus_expfmt::format::Format; @@ -60,6 +61,7 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { // Wrap `ArrowDecoder` in `!Send` `Rc>` to mirror thread-per-core OTAP receivers. let shared_decoder = Rc::new(RefCell::new(ArrowDecoder::new())); let opts = DecodeOptions::new().with_timestamp_ms(1_700_000_000_000); + let opts_for_full = opts.clone(); let full_237_clone = Rc::clone(&full_237); let handle = tokio::task::spawn_local(async move { @@ -70,7 +72,7 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { for i in 0..32 { let records = shared_decoder .borrow_mut() - .decode(&full_237_clone, &Format::TEXT, &opts) + .decode(&full_237_clone, &Format::TEXT, &opts_for_full) .expect("scrape succeeds"); let otap = to_otap_arrow_records(&records); let dec_mem = shared_decoder.borrow().retained_memory_bytes(); @@ -113,7 +115,8 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { // Verify `decode_stream` caps retained decoder scratch memory to a single 128-point chunk // and keeps both decoder capacity and live heap constant across repeated scrapes. let stream_decoder = Rc::new(RefCell::new(ArrowDecoder::new())); - let stream_opts = opts.with_max_data_points_per_batch(128); + let stream_opts = opts.clone().with_max_data_points_per_batch(128); + let stream_opts_for_coop = stream_opts.clone(); let full_237_coop = Rc::clone(&full_237); let stream_handle = tokio::task::spawn_local(async move { let mut warmup_stream_dec_mem = 0usize; @@ -174,7 +177,7 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { for i in 0..32 { let mut cursor = coop_decoder .borrow_mut() - .start_decode_stream(&Format::TEXT, &stream_opts) + .start_decode_stream(&Format::TEXT, &stream_opts_for_coop) .expect("cursor starts"); loop { let step = coop_decoder @@ -196,7 +199,7 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { let mut stream = owned_coop_decoder.decode_stream( &full_237_coop, &Format::TEXT, - &stream_opts, + &stream_opts_for_coop, ); while let Some(chunk_res) = stream .next_cooperative(coop_budget, tokio::task::yield_now) @@ -241,10 +244,17 @@ fn local_set_non_send_repeated_scrapes_keep_bounded_memory() { .await .expect("cooperative streaming local task succeeds"); - // Verify OpenMetrics 1.0 scrapes (with histograms, summaries, exemplars, and units) also - // keep retained scratch memory and live heap strictly constant after warmup. + // Verify OpenMetrics 1.0 scrapes (with histograms, summaries, exemplars, units, and cached + // target `ResourceAttrs`) also keep retained scratch memory and live heap strictly constant + // after warmup. let om_decoder = Rc::new(RefCell::new(ArrowDecoder::new())); - let om_opts = opts; + let target_attrs = Arc::new( + TargetAttributes::new() + .with_job("om-target") + .with_instance("10.0.0.1:9090") + .with_target_label("env", "prod"), + ); + let om_opts = opts.with_target_attributes(target_attrs); let om_handle = tokio::task::spawn_local(async move { let mut warmup_om_dec_mem = 0usize; let mut warmup_om_live_heap = 0usize;