Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions crates/expfmt/benches/decode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ fn main() {
let full_bytes = full.as_bytes();

let opts = DecodeOptions::new().with_timestamp_ms(0);
let stream_256_opts = opts.with_max_data_points_per_batch(256);
let stream_256_opts = opts.clone().with_max_data_points_per_batch(256);
let budget = DecodeBudget::default();

let mut suite = Suite::from_env("prometheus-expfmt").with_alloc(&ALLOC);
Expand All @@ -121,19 +121,28 @@ fn main() {
("alltypes.237mfs", full_bytes),
] {
let bytes = payload.len() as u64;
let opts_decode = opts.clone();
let opts_coop = opts.clone();
let opts_stream_16k = opts.clone();
let opts_stream_256 = stream_256_opts.clone();
let opts_stream_256_coop = stream_256_opts.clone();

suite = suite
.bench(format!("Decode/case={case}/mode=decode"), move |b| {
// Warm internal staging and interner capacities once so B/op and allocs/op
// measure steady-state scrape reuse rather than cold capacity growth divided by N.
let mut decoder = ArrowDecoder::new();
black_box(decoder.decode(payload, &Format::TEXT, &opts).unwrap());
black_box(
decoder
.decode(payload, &Format::TEXT, &opts_decode)
.unwrap(),
);

b.set_bytes(bytes);
b.report_allocs();
while b.keep_running() {
let records = decoder
.decode(black_box(payload), &Format::TEXT, &opts)
.decode(black_box(payload), &Format::TEXT, &opts_decode)
.unwrap();
black_box(records);
}
Expand All @@ -142,15 +151,15 @@ fn main() {
format!("Decode/case={case}/mode=decode_cooperative"),
move |b| {
let mut decoder = ArrowDecoder::new();
black_box(decoder.decode(payload, &Format::TEXT, &opts).unwrap());
black_box(decoder.decode(payload, &Format::TEXT, &opts_coop).unwrap());

b.set_bytes(bytes);
b.report_allocs();
while b.keep_running() {
let records = poll_cooperative(decoder.decode_cooperative(
black_box(payload),
&Format::TEXT,
&opts,
&opts_coop,
budget,
|| YieldOnce(false),
))
Expand All @@ -161,29 +170,31 @@ fn main() {
)
.bench(format!("Decode/case={case}/mode=stream_16k"), move |b| {
let mut decoder = ArrowDecoder::new();
for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts) {
for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts_stream_16k) {
black_box(chunk.unwrap());
}

b.set_bytes(bytes);
b.report_allocs();
while b.keep_running() {
for chunk in decoder.decode_stream(black_box(payload), &Format::TEXT, &opts) {
for chunk in
decoder.decode_stream(black_box(payload), &Format::TEXT, &opts_stream_16k)
{
black_box(chunk.unwrap());
}
}
})
.bench(format!("Decode/case={case}/mode=stream_256"), move |b| {
let mut decoder = ArrowDecoder::new();
for chunk in decoder.decode_stream(payload, &Format::TEXT, &stream_256_opts) {
for chunk in decoder.decode_stream(payload, &Format::TEXT, &opts_stream_256) {
black_box(chunk.unwrap());
}

b.set_bytes(bytes);
b.report_allocs();
while b.keep_running() {
for chunk in
decoder.decode_stream(black_box(payload), &Format::TEXT, &stream_256_opts)
decoder.decode_stream(black_box(payload), &Format::TEXT, &opts_stream_256)
{
black_box(chunk.unwrap());
}
Expand All @@ -193,7 +204,9 @@ fn main() {
format!("Decode/case={case}/mode=stream_256_cooperative"),
move |b| {
let mut decoder = ArrowDecoder::new();
for chunk in decoder.decode_stream(payload, &Format::TEXT, &stream_256_opts) {
for chunk in
decoder.decode_stream(payload, &Format::TEXT, &opts_stream_256_coop)
{
black_box(chunk.unwrap());
}

Expand All @@ -203,7 +216,7 @@ fn main() {
let mut stream = decoder.decode_stream(
black_box(payload),
&Format::TEXT,
&stream_256_opts,
&opts_stream_256_coop,
);
loop {
match stream.step(budget).unwrap() {
Expand Down
29 changes: 19 additions & 10 deletions crates/expfmt/src/arrow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ use self::intern::StringInterner;
pub use self::options::{
CounterSuffixPolicy, DECODE_BUDGET_DURATION_CHECK_INTERVAL, DEFAULT_DECODE_BUDGET_MAX_BYTES,
DEFAULT_DECODE_BUDGET_MAX_DURATION, DEFAULT_DECODE_BUDGET_MAX_ENTRIES,
DEFAULT_MAX_DATA_POINTS_PER_BATCH, DecodeBudget, DecodeError, DecodeOptions, UnitSuffixPolicy,
UntypedPolicy,
DEFAULT_MAX_DATA_POINTS_PER_BATCH, DecodeBudget, DecodeError, DecodeOptions, TargetAttributes,
UnitSuffixPolicy, UntypedPolicy,
};
pub use self::records::{MetricPayloadType, OtapMetricRecords};
pub use self::schema::PROMETHEUS_TYPE_METADATA_KEY;
pub use self::schema::{PROMETHEUS_TYPE_METADATA_KEY, SERVICE_INSTANCE_ID_ATTR, SERVICE_NAME_ATTR};

/// Maximum distinct `UnivariateMetrics` rows representable in a single OTAP batch (`u16` IDs).
const MAX_METRICS_PER_BATCH: usize = (u16::MAX as usize) + 1;
Expand Down Expand Up @@ -654,7 +654,7 @@ impl ArrowDecoder {
input,
cursor: DecodeCursor {
parser_state: ActiveParserState::Invalid,
opts: *opts,
opts: opts.clone(),
default_ts_ns: None,
max_data_points,
scrape_id,
Expand Down Expand Up @@ -686,7 +686,7 @@ impl ArrowDecoder {
let (parser, default_ts_ns) = self.init_scrape_parser(b"", format, opts)?;
Ok(DecodeCursor {
parser_state: parser.into_state(),
opts: *opts,
opts: opts.clone(),
default_ts_ns,
max_data_points,
scrape_id: self.scrape_id,
Expand Down Expand Up @@ -844,7 +844,7 @@ impl ArrowDecoder {
if self.metrics.is_empty() {
return Ok(DecodeStep::Done);
}
let batch = self.finish_batches(default_ts_ns.is_some())?;
let batch = self.finish_batches(opts, default_ts_ns.is_some())?;
if CHUNKED {
self.reset_batch();
}
Expand Down Expand Up @@ -892,7 +892,7 @@ impl ArrowDecoder {
self.current_family
.start_new(raw_name, &self.series_name_buf);
*pending_series = true;
let batch = self.finish_batches(default_ts_ns.is_some())?;
let batch = self.finish_batches(opts, default_ts_ns.is_some())?;
self.reset_batch();
return Ok(DecodeStep::Batch(batch));
}
Expand All @@ -915,7 +915,7 @@ impl ArrowDecoder {
self.flush_pending_family(opts, default_ts_ns)?;
}
*pending_series = true;
let batch = self.finish_batches(default_ts_ns.is_some())?;
let batch = self.finish_batches(opts, default_ts_ns.is_some())?;
self.reset_batch();
return Ok(DecodeStep::Batch(batch));
}
Expand All @@ -927,7 +927,7 @@ impl ArrowDecoder {
&& self.assembly.is_empty()
&& self.total_data_points() >= max_data_points
{
let batch = self.finish_batches(default_ts_ns.is_some())?;
let batch = self.finish_batches(opts, default_ts_ns.is_some())?;
self.reset_batch();
return Ok(DecodeStep::Batch(batch));
}
Expand Down Expand Up @@ -1549,11 +1549,19 @@ impl ArrowDecoder {
Ok(())
}

fn finish_batches(&mut self, has_default_ts: bool) -> Result<OtapMetricRecords, DecodeError> {
fn finish_batches(
&mut self,
opts: &DecodeOptions,
has_default_ts: bool,
) -> Result<OtapMetricRecords, DecodeError> {
if self.metrics.is_empty() {
return Ok(OtapMetricRecords::default());
}

let resource_attrs = match opts.target_attributes.as_deref() {
Some(target) => target.resource_attrs_batch()?,
None => None,
};
let univariate_metrics = Some(self.metrics.build_univariate_metrics(&mut self.scratch)?);
let number_data_points = self.num_dp.build_batch(has_default_ts, &mut self.scratch)?;
let summary_data_points = self.sum_dp.build_batch(has_default_ts, &mut self.scratch)?;
Expand Down Expand Up @@ -1589,6 +1597,7 @@ impl ArrowDecoder {
let metric_attrs = Some(self.metrics.build_metric_attrs(&mut self.scratch)?);

Ok(OtapMetricRecords {
resource_attrs,
univariate_metrics,
metric_attrs,
number_data_points,
Expand Down
56 changes: 54 additions & 2 deletions crates/expfmt/src/arrow/columns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

//! Reusable zero-allocation column and table staging buffers for [`super::ArrowDecoder`].

use std::collections::BTreeMap;
use std::sync::Arc;

use arrow_array::types::{UInt8Type, UInt16Type};
Expand All @@ -30,8 +31,9 @@ use super::schema::{
OTAP_METRIC_TYPE_HISTOGRAM, OTAP_METRIC_TYPE_SUM, UTC_TIME_ZONE, ValEncoding, dp_attrs_schema,
exemplars_schema, histogram_bucket_counts_item_field, histogram_dp_schema,
histogram_explicit_bounds_item_field, metric_attrs_key_dict, metric_attrs_schema,
metric_attrs_type_val_dict, number_dp_schema, resource_struct_fields, summary_dp_schema,
summary_quantile_item_field, summary_quantile_struct_fields, univariate_metrics_schema,
metric_attrs_type_val_dict, number_dp_schema, resource_attrs_schema, resource_struct_fields,
summary_dp_schema, summary_quantile_item_field, summary_quantile_struct_fields,
univariate_metrics_schema,
};

/// Reusable scratch vectors used when materializing Arrow columns without reallocating.
Expand Down Expand Up @@ -748,6 +750,56 @@ impl AttrTable {
}
}

/// Builds an OTAP `ResourceAttrs` (`attributes_16`) [`RecordBatch`] attached to `parent_id = 0`
/// (matching `UnivariateMetrics.resource.id = 0`).
///
/// Returns `Ok(None)` when `attrs` is empty so empty tables are omitted per OTAP invariants.
pub(super) fn build_resource_attrs_batch(
attrs: &BTreeMap<String, String>,
) -> Result<Option<RecordBatch>, DecodeError> {
if attrs.is_empty() {
return Ok(None);
}

let num_attrs = attrs.len();
let key_bytes: usize = attrs.keys().map(String::len).sum();
let val_bytes: usize = attrs.values().map(String::len).sum();
let mut key_interner = StringInterner::with_capacity(num_attrs, key_bytes);
let mut val_interner = StringInterner::with_capacity(num_attrs, val_bytes);
let mut pairs = Vec::with_capacity(num_attrs);

for (k, v) in attrs {
let kid =
key_interner
.intern(k.as_bytes())
.ok_or_else(|| DecodeError::CapacityOverflow {
table: "ResourceAttrs.key",
count: key_interner.len(),
})?;
let vid =
val_interner
.intern(v.as_bytes())
.ok_or_else(|| DecodeError::CapacityOverflow {
table: "ResourceAttrs.str",
count: val_interner.len(),
})?;
pairs.push((kid, vid));
}

let enc = AttrEncoding::new(&key_interner, &val_interner, true);
let mut scratch = ScratchBuffers::with_capacity(0, 0, num_attrs);
scratch.u16_buf.clear();
scratch.u16_buf.resize(num_attrs, 0);
let parent_id_arr: ArrayRef =
Arc::new(UInt16Array::new(scalar_from_slice(&scratch.u16_buf), None));
let (key_arr, type_arr, val_arr) = scratch.build_attr_columns(&pairs, &enc, "ResourceAttrs")?;

Ok(Some(RecordBatch::try_new(
resource_attrs_schema(enc.key_enc, enc.val_enc),
vec![parent_id_arr, key_arr, type_arr, val_arr],
)?))
}

/// Staging buffer for an optional fixed-size binary column (`span_id` or `trace_id` on exemplars).
#[derive(Default)]
pub(super) struct SparseFixedBinaryCol<const N: usize> {
Expand Down
Loading
Loading