From 0848bbd27cf3fcceb11c049d8b02920eed4567be Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Wed, 18 Nov 2026 05:38:10 +0000 Subject: [PATCH 1/2] scrape-manager: wire target discovery relabeling, honor_labels, and metric_relabel_configs Add metrics_path, scheme, params, honor_labels, relabel_configs, and metric_relabel_configs to ScrapeConfig, along with ScrapeConfig::populate_targets and ScrapeTarget::{from_populated, from_populated_with_params} matching Prometheus Target.URL() parameter merging. Compile metric_relabel_configs once into Arc inside ScrapeManager::new and attach honor_labels and cached TargetAttributes per target during Arrow decoding while keeping ScrapeTarget free of interior mutability (per-slot OnceLock in TargetSchedule and a bounded cache for direct scrape_once_arrow calls). Signed-off-by: Bartek Plotka --- crates/scrape-manager/src/scrape.rs | 5 +- crates/scrape-manager/src/scrape/config.rs | 270 ++++++++++++++++++- crates/scrape-manager/src/scrape/manager.rs | 129 +++++++-- crates/scrape-manager/src/scrape/schedule.rs | 55 +++- 4 files changed, 422 insertions(+), 37 deletions(-) diff --git a/crates/scrape-manager/src/scrape.rs b/crates/scrape-manager/src/scrape.rs index 475cdce..4e907ec 100644 --- a/crates/scrape-manager/src/scrape.rs +++ b/crates/scrape-manager/src/scrape.rs @@ -32,8 +32,9 @@ mod schedule; pub use config::{ DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_DATA_POINTS_PER_BATCH, DEFAULT_MAX_RETAINED_BODY_BYTES, - DEFAULT_MAX_RETAINED_DECODER_BYTES, DEFAULT_SCRAPE_INTERVAL, DEFAULT_SCRAPE_TIMEOUT, - DEFAULT_USER_AGENT, SchedulingMode, ScrapeConfig, ScrapeTarget, SequentialSchedule, + DEFAULT_MAX_RETAINED_DECODER_BYTES, DEFAULT_METRICS_PATH, DEFAULT_SCHEME, + DEFAULT_SCRAPE_INTERVAL, DEFAULT_SCRAPE_TIMEOUT, DEFAULT_USER_AGENT, SchedulingMode, + ScrapeConfig, ScrapeTarget, SequentialSchedule, }; pub use http::{ FetchedScrape, ReqwestFetcher, ScrapeError, TargetFetcher, build_accept_header, diff --git a/crates/scrape-manager/src/scrape/config.rs b/crates/scrape-manager/src/scrape/config.rs index 0b93c7f..8e13611 100644 --- a/crates/scrape-manager/src/scrape/config.rs +++ b/crates/scrape-manager/src/scrape/config.rs @@ -16,9 +16,14 @@ use std::num::NonZeroUsize; use std::time::Duration; #[cfg(feature = "arrow")] -use prometheus_expfmt::arrow::{DecodeBudget, DecodeOptions}; +use prometheus_expfmt::arrow::{DecodeBudget, DecodeOptions, TargetAttributes}; use prometheus_expfmt::format::EscapingScheme; +use prometheus_relabel::{ + ADDRESS_LABEL, Config as RelabelConfig, METRICS_PATH_LABEL, PARAM_LABEL_PREFIX, + PopulatedTarget, SCHEME_LABEL, TargetPopulateConfig, ValidationScheme, populate_labels, +}; +use crate::discovery::{TargetGroup, format_duration}; use crate::scrape::ScrapeError; /// Default interval between scrapes of the same target (`15s`). @@ -27,6 +32,12 @@ pub const DEFAULT_SCRAPE_INTERVAL: Duration = Duration::from_secs(15); /// Default per-target HTTP scrape timeout (`10s`). pub const DEFAULT_SCRAPE_TIMEOUT: Duration = Duration::from_secs(10); +/// Default HTTP request path scraped on each target (`"/metrics"`). +pub const DEFAULT_METRICS_PATH: &str = "/metrics"; + +/// Default URL scheme scraped on each target (`"http"`). +pub const DEFAULT_SCHEME: &str = "http"; + /// Default maximum uncompressed HTTP response body size (`16 MiB`). pub const DEFAULT_MAX_BODY_BYTES: usize = 16 * 1024 * 1024; @@ -82,6 +93,101 @@ impl ScrapeTarget { Ok(Self::new(url)) } + /// Builds a [`ScrapeTarget`] from a relabeled [`PopulatedTarget`], constructing the scrape URL + /// from `__scheme__`, `__address__`, `__metrics_path__`, and `__param_*` labels and storing + /// only the public (non-`__`-prefixed) target labels. + /// + /// # Errors + /// + /// Returns [`ScrapeError::InvalidConfig`] if `__scheme__` is missing or not `http`/`https`, + /// or if `__address__` cannot form a valid target URL. + pub fn from_populated(populated: &PopulatedTarget) -> Result { + Self::from_populated_with_params(populated, &BTreeMap::new()) + } + + /// Builds a [`ScrapeTarget`] from a relabeled [`PopulatedTarget`] and optional base + /// `scrape_config.params`, matching Prometheus `Target.URL()` query parameter merging + /// (`__param_` replaces the first value of `config_params[name]` while preserving any + /// additional values). + /// + /// # Errors + /// + /// Returns [`ScrapeError::InvalidConfig`] if `__scheme__` is missing or not `http`/`https`, + /// or if `__address__` cannot form a valid target URL. + pub fn from_populated_with_params( + populated: &PopulatedTarget, + config_params: &BTreeMap>, + ) -> Result { + let scheme = populated + .labels + .get(SCHEME_LABEL) + .map(String::as_str) + .unwrap_or(""); + if !matches!(scheme, "http" | "https") { + return Err(ScrapeError::InvalidConfig(format!( + "unsupported scrape target scheme {scheme:?}" + ))); + } + + let address = populated + .labels + .get(ADDRESS_LABEL) + .map(String::as_str) + .unwrap_or(""); + if address.is_empty() || address.contains('/') { + return Err(ScrapeError::InvalidConfig(format!( + "invalid scrape target address {address:?}" + ))); + } + + let mut url = reqwest::Url::parse(&format!("{scheme}://{address}")).map_err(|err| { + ScrapeError::InvalidConfig(format!( + "invalid scrape target URL from scheme {scheme:?} and address {address:?}: {err}" + )) + })?; + if url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(ScrapeError::InvalidConfig(format!( + "invalid scrape target address {address:?}" + ))); + } + + if let Some(path) = populated.labels.get(METRICS_PATH_LABEL) { + url.set_path(path); + } + + let mut params = config_params.clone(); + for (key, value) in &populated.labels { + let Some(param_name) = key.strip_prefix(PARAM_LABEL_PREFIX) else { + continue; + }; + if let Some(existing) = params.get_mut(param_name) + && !existing.is_empty() + { + existing[0].clone_from(value); + } else { + params.insert(param_name.to_owned(), vec![value.clone()]); + } + } + if !params.is_empty() { + let mut pairs = url.query_pairs_mut(); + for (key, values) in ¶ms { + for value in values { + pairs.append_pair(key, value); + } + } + } + + Ok(Self { + url, + labels: populated.public_labels(), + }) + } + /// Inserts or replaces a target metadata label (such as `instance` or `job`). #[must_use] pub fn with_label(mut self, key: impl Into, value: impl Into) -> Self { @@ -114,6 +220,18 @@ impl ScrapeTarget { pub const fn labels(&self) -> &BTreeMap { &self.labels } + + /// Builds OpenTelemetry [`TargetAttributes`] derived from [`Self::labels`], returning `None` + /// when the target has no public non-empty labels. + #[cfg(feature = "arrow")] + #[must_use] + pub fn to_target_attributes(&self) -> Option { + if self.labels.is_empty() { + return None; + } + let attrs = TargetAttributes::from_target_labels(&self.labels); + (!attrs.is_empty()).then_some(attrs) + } } /// Cadence policy when [`SchedulingMode::Sequential`] is selected. @@ -286,6 +404,22 @@ pub struct ScrapeConfig { pub max_data_points_per_batch: usize, /// `User-Agent` header value sent on HTTP scrape requests. pub user_agent: String, + /// Default HTTP path populated into `__metrics_path__` during target discovery + /// (defaults to `"/metrics"`). + pub metrics_path: String, + /// Default URL scheme populated into `__scheme__` during target discovery (`"http"` or + /// `"https"`, defaults to `"http"`). + pub scheme: String, + /// Optional HTTP query parameters populated into `__param_` and appended to target URLs. + pub params: BTreeMap>, + /// Controls how exposed sample labels colliding with public target labels are resolved, + /// matching Prometheus `scrape_config.honor_labels` (defaults to `false`). + pub honor_labels: bool, + /// Target discovery relabeling rules applied by [`ScrapeConfig::populate_targets`]. + pub relabel_configs: Vec, + /// Post-scrape metric relabeling rules compiled once into a `MetricRelabelPlan` by + /// [`crate::scrape::ScrapeManager::new`] and executed in `ArrowDecoder`. + pub metric_relabel_configs: Vec, /// Cooperative per-step parsing budget when the `arrow` feature is enabled. #[cfg(feature = "arrow")] pub decode_budget: DecodeBudget, @@ -311,6 +445,12 @@ impl Default for ScrapeConfig { max_retained_decoder_bytes: DEFAULT_MAX_RETAINED_DECODER_BYTES, max_data_points_per_batch: DEFAULT_MAX_DATA_POINTS_PER_BATCH, user_agent: DEFAULT_USER_AGENT.to_owned(), + metrics_path: DEFAULT_METRICS_PATH.to_owned(), + scheme: DEFAULT_SCHEME.to_owned(), + params: BTreeMap::new(), + honor_labels: false, + relabel_configs: Vec::new(), + metric_relabel_configs: Vec::new(), #[cfg(feature = "arrow")] decode_budget: DecodeBudget::default(), #[cfg(feature = "arrow")] @@ -420,6 +560,55 @@ impl ScrapeConfig { self } + /// Sets [`ScrapeConfig::metrics_path`]. + #[must_use] + pub fn with_metrics_path(mut self, path: impl Into) -> Self { + self.metrics_path = path.into(); + self + } + + /// Sets [`ScrapeConfig::scheme`]. + #[must_use] + pub fn with_scheme(mut self, scheme: impl Into) -> Self { + self.scheme = scheme.into(); + self + } + + /// Sets [`ScrapeConfig::params`]. + #[must_use] + pub fn with_params(mut self, params: BTreeMap>) -> Self { + self.params = params; + self + } + + /// Inserts or replaces a single query parameter in [`ScrapeConfig::params`]. + #[must_use] + pub fn with_param(mut self, key: impl Into, values: Vec) -> Self { + self.params.insert(key.into(), values); + self + } + + /// Sets [`ScrapeConfig::honor_labels`]. + #[must_use] + pub const fn with_honor_labels(mut self, honor_labels: bool) -> Self { + self.honor_labels = honor_labels; + self + } + + /// Sets [`ScrapeConfig::relabel_configs`]. + #[must_use] + pub fn with_relabel_configs(mut self, configs: Vec) -> Self { + self.relabel_configs = configs; + self + } + + /// Sets [`ScrapeConfig::metric_relabel_configs`]. + #[must_use] + pub fn with_metric_relabel_configs(mut self, configs: Vec) -> Self { + self.metric_relabel_configs = configs; + self + } + /// Sets [`ScrapeConfig::decode_budget`]. #[cfg(feature = "arrow")] #[must_use] @@ -436,13 +625,66 @@ impl ScrapeConfig { self } - /// Validates interval, timeout, and pool invariants. + /// Builds the [`TargetPopulateConfig`] used to seed pre-relabel target discovery labels. + #[must_use] + pub fn target_populate_config(&self) -> TargetPopulateConfig { + TargetPopulateConfig { + job_name: self.job_name.clone(), + scrape_interval: format_duration(self.scrape_interval), + scrape_timeout: format_duration(self.scrape_timeout), + metrics_path: self.metrics_path.clone(), + scheme: self.scheme.clone(), + params: self.params.clone(), + } + } + + /// Populates and relabels all discovered targets across `groups` using + /// [`Self::relabel_configs`], returning the kept [`ScrapeTarget`] endpoints ready for + /// [`crate::scrape::ScrapeManager`]. + /// + /// Targets dropped by [`Self::relabel_configs`] are omitted from the returned vector. Pool + /// scheduling uses [`Self::scrape_interval`] and [`Self::scrape_timeout`]. + /// + /// # Errors + /// + /// Returns [`ScrapeError::InvalidConfig`] if `self` fails validation, if `populate_labels` + /// rejects a relabeled target (for example missing `__address__` or invalid + /// `__scrape_interval__`/`__scrape_timeout__`), or if the relabeled target cannot form a + /// valid HTTP/HTTPS URL. + pub fn populate_targets( + &self, + groups: &[TargetGroup], + ) -> Result, ScrapeError> { + self.validate()?; + let validated_relabel = validate_relabel_configs(&self.relabel_configs)?; + let pop_cfg = self.target_populate_config(); + + let mut out = Vec::new(); + for group in groups { + for target_labels in &group.targets { + let Some(populated) = + populate_labels(target_labels, &group.labels, &pop_cfg, &validated_relabel) + .map_err(|err| ScrapeError::InvalidConfig(err.to_string()))? + else { + continue; + }; + out.push(ScrapeTarget::from_populated_with_params( + &populated, + &self.params, + )?); + } + } + Ok(out) + } + + /// Validates interval, timeout, scheme, relabel rules, and pool invariants. /// /// # Errors /// /// Returns [`ScrapeError::InvalidConfig`] if `job_name` is empty, `scrape_interval` or - /// `scrape_timeout` is zero, `scrape_timeout > scrape_interval`, or a concurrent pool size - /// exceeds `max_in_flight`. + /// `scrape_timeout` is zero, `scrape_timeout > scrape_interval`, `scheme` is not `"http"` or + /// `"https"`, any rule in `relabel_configs` or `metric_relabel_configs` is invalid, or a + /// concurrent pool size exceeds `max_in_flight`. pub fn validate(&self) -> Result<(), ScrapeError> { if self.job_name.trim().is_empty() { return Err(ScrapeError::InvalidConfig( @@ -477,12 +719,20 @@ impl ScrapeConfig { self.scrape_timeout, self.scrape_interval ))); } + if !matches!(self.scheme.as_str(), "http" | "https") { + return Err(ScrapeError::InvalidConfig(format!( + "unsupported scrape scheme {:?} (expected \"http\" or \"https\")", + self.scheme + ))); + } if reqwest::header::HeaderValue::from_str(&self.user_agent).is_err() { return Err(ScrapeError::InvalidConfig(format!( "user_agent contains invalid HTTP header bytes: {:?}", self.user_agent ))); } + let _ = validate_relabel_configs(&self.relabel_configs)?; + let _ = validate_relabel_configs(&self.metric_relabel_configs)?; if let SchedulingMode::Concurrent { max_in_flight, max_buffered_bodies, @@ -514,6 +764,18 @@ impl ScrapeConfig { } } +fn validate_relabel_configs(cfgs: &[RelabelConfig]) -> Result, ScrapeError> { + let mut validated = Vec::with_capacity(cfgs.len()); + for cfg in cfgs { + let mut cloned = cfg.clone(); + cloned + .validate(ValidationScheme::Utf8) + .map_err(|err| ScrapeError::InvalidConfig(err.to_string()))?; + validated.push(cloned); + } + Ok(validated) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/scrape-manager/src/scrape/manager.rs b/crates/scrape-manager/src/scrape/manager.rs index aec1863..a763e20 100644 --- a/crates/scrape-manager/src/scrape/manager.rs +++ b/crates/scrape-manager/src/scrape/manager.rs @@ -11,9 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "arrow")] +use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::ops::ControlFlow; use std::sync::Arc; +#[cfg(feature = "arrow")] +use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; #[cfg(feature = "arrow")] @@ -22,7 +26,11 @@ use std::time::Instant as StdInstant; use futures_util::StreamExt; use futures_util::stream::FuturesUnordered; #[cfg(feature = "arrow")] -use prometheus_expfmt::arrow::{DecodeBudget, DecodeCursor, DecodeStep, OtapMetricRecords}; +use prometheus_expfmt::arrow::{ + DecodeBudget, DecodeCursor, DecodeStep, MetricRelabelPlan, OtapMetricRecords, TargetAttributes, +}; +#[cfg(feature = "arrow")] +use prometheus_relabel::ValidationScheme; use tokio::sync::mpsc; use tokio::time::{Instant, sleep_until}; @@ -30,12 +38,20 @@ use tokio::time::{Instant, sleep_until}; use crate::scrape::DecoderPool; #[cfg(feature = "arrow")] use crate::scrape::http::current_timestamp_ms; -use crate::scrape::schedule::{TargetSchedule, advance_grid_deadline}; +use crate::scrape::schedule::{ScheduledTarget, TargetSchedule, advance_grid_deadline}; use crate::scrape::{ BodyBufferPool, FetchedScrape, ReqwestFetcher, SchedulingMode, ScrapeConfig, ScrapeError, ScrapeTarget, SequentialSchedule, TargetFetcher, }; +/// Upper bound on cached one-shot [`TargetAttributes`] entries kept for repeated +/// `scrape_once_arrow` / `run_once_arrow` calls so unbounded target churn cannot leak memory. +#[cfg(feature = "arrow")] +const MAX_CACHED_TARGET_ATTRS: usize = 1_024; + +#[cfg(feature = "arrow")] +type TargetAttrsCache = HashMap, Option>>; + /// Control-plane messages accepted by a running [`ScrapeManager`]. #[derive(Clone, Debug)] #[non_exhaustive] @@ -216,6 +232,10 @@ pub struct ScrapeManager { body_pool: BodyBufferPool, #[cfg(feature = "arrow")] decoder_pool: DecoderPool, + #[cfg(feature = "arrow")] + metric_relabel_plan: Option>, + #[cfg(feature = "arrow")] + target_attrs_cache: Mutex, counters: AtomicCounters, } @@ -255,12 +275,28 @@ impl ScrapeManager { config.scheduling.max_decoders(), config.max_retained_decoder_bytes, ); + #[cfg(feature = "arrow")] + let metric_relabel_plan = if config.metric_relabel_configs.is_empty() { + None + } else { + Some(Arc::new( + MetricRelabelPlan::try_new( + config.metric_relabel_configs.clone(), + ValidationScheme::Utf8, + ) + .map_err(|err| ScrapeError::InvalidConfig(err.to_string()))?, + )) + }; Ok(Self { config, fetcher, body_pool, #[cfg(feature = "arrow")] decoder_pool, + #[cfg(feature = "arrow")] + metric_relabel_plan, + #[cfg(feature = "arrow")] + target_attrs_cache: Mutex::new(HashMap::new()), counters: AtomicCounters::default(), }) } @@ -334,20 +370,53 @@ impl ScrapeManager { ctrl_rx: mpsc::UnboundedReceiver, sink: &S, ) { - self.run_loop(initial_targets, ctrl_rx, |target| async move { - let res = self.scrape_once_raw(&target).await; - sink.on_raw_scrape(&target, res).await + self.run_loop(initial_targets, ctrl_rx, |scheduled| async move { + let res = self.scrape_once_raw(&scheduled.target).await; + sink.on_raw_scrape(&scheduled.target, res).await }) .await; } - /// Fetches `target` once, streams cooperatives chunks through a pooled [`ArrowDecoder`], and - /// emits a final [`ScrapeReport`] to `sink`. + /// Fetches `target` once, streams cooperative chunks through a pooled `ArrowDecoder` with + /// target `ResourceAttrs`, `honor_labels`, and compiled `metric_relabel_configs`, and emits a + /// final [`ScrapeReport`] to `sink`. #[cfg(feature = "arrow")] pub async fn scrape_once_arrow( &self, target: &ScrapeTarget, sink: &S, + ) -> ControlFlow<(), ScrapeReport> { + let target_attrs = self.cached_target_attributes(target); + self.scrape_once_arrow_with_attrs(target, target_attrs, sink) + .await + } + + #[cfg(feature = "arrow")] + fn cached_target_attributes(&self, target: &ScrapeTarget) -> Option> { + if target.labels().is_empty() { + return None; + } + let mut cache = self + .target_attrs_cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(cached) = cache.get(target.labels()) { + return cached.clone(); + } + let built = target.to_target_attributes().map(Arc::new); + if cache.len() >= MAX_CACHED_TARGET_ATTRS { + cache.clear(); + } + cache.insert(target.labels().clone(), built.clone()); + built + } + + #[cfg(feature = "arrow")] + async fn scrape_once_arrow_with_attrs( + &self, + target: &ScrapeTarget, + target_attrs: Option>, + sink: &S, ) -> ControlFlow<(), ScrapeReport> { let started = StdInstant::now(); let started_ms = current_timestamp_ms(); @@ -383,7 +452,7 @@ impl ScrapeManager { let scrape_timestamp_ms = fetched.scrape_timestamp_ms; let decode_outcome = self - .decode_fetched_cooperatively(target, fetched, sink) + .decode_fetched_cooperatively(target, target_attrs, fetched, sink) .await; let report = match decode_outcome { @@ -455,8 +524,12 @@ impl ScrapeManager { ctrl_rx: mpsc::UnboundedReceiver, sink: &S, ) { - self.run_loop(initial_targets, ctrl_rx, |target| async move { - match self.scrape_once_arrow(&target, sink).await { + self.run_loop(initial_targets, ctrl_rx, |scheduled| async move { + let target_attrs = scheduled.target_attributes(); + match self + .scrape_once_arrow_with_attrs(&scheduled.target, target_attrs, sink) + .await + { ControlFlow::Continue(_) => ControlFlow::Continue(()), ControlFlow::Break(()) => ControlFlow::Break(()), } @@ -468,6 +541,7 @@ impl ScrapeManager { async fn decode_fetched_cooperatively( &self, target: &ScrapeTarget, + target_attrs: Option>, fetched: FetchedScrape, sink: &S, ) -> ControlFlow<(), (u64, usize, Result<(), ScrapeError>)> { @@ -504,12 +578,19 @@ impl ScrapeManager { // TODO(bwplotka): Add and wire `ScrapeConfig::honor_timestamps` once // `DecodeOptions::with_honor_timestamps` lands in // https://github.com/prometheus/scrape.rs/issues/33. - let decode_opts = self + let mut decode_opts = self .config .decode_options .clone() .with_timestamp_ms(scrape_timestamp_ms) - .with_max_data_points_per_batch(chunk_limit); + .with_max_data_points_per_batch(chunk_limit) + .with_honor_labels(self.config.honor_labels || self.config.decode_options.honor_labels); + if let Some(target_attrs) = target_attrs { + decode_opts = decode_opts.with_target_attributes(target_attrs); + } + if let Some(plan) = &self.metric_relabel_plan { + decode_opts = decode_opts.with_metric_relabel_plan(Arc::clone(plan)); + } let cursor = match decoder.start_decode_stream(&format, &decode_opts) { Ok(c) => c, @@ -685,7 +766,7 @@ impl ScrapeManager { ctrl_rx: mpsc::UnboundedReceiver, exec_one: Exec, ) where - Exec: Fn(Arc) -> Fut, + Exec: Fn(ScheduledTarget) -> Fut, Fut: Future>, { match self.config.scheduling { @@ -714,11 +795,13 @@ impl ScrapeManager { mut ctrl_rx: mpsc::UnboundedReceiver, exec_one: Exec, ) where - Exec: Fn(Arc) -> Fut, + Exec: Fn(ScheduledTarget) -> Fut, Fut: Future>, { - let mut targets: Vec> = - initial_targets.into_iter().map(Arc::new).collect(); + let mut targets: Vec = initial_targets + .into_iter() + .map(|t| ScheduledTarget::new(Arc::new(t))) + .collect(); let interval = self.config.scrape_interval; let start_now = Instant::now(); let mut next_due = start_now @@ -732,7 +815,10 @@ impl ScrapeManager { cmd = ctrl_rx.recv() => { match cmd { Some(ScrapeCommand::UpdateTargets(new_targets)) => { - targets = new_targets.into_iter().map(Arc::new).collect(); + targets = new_targets + .into_iter() + .map(|t| ScheduledTarget::new(Arc::new(t))) + .collect(); } Some(ScrapeCommand::Shutdown) | None => return, } @@ -763,7 +849,10 @@ impl ScrapeManager { // Stage the updated target list for the next cycle // tick without cancelling the in-flight scrape or // skipping remaining targets in the current pass. - targets = new_targets.into_iter().map(Arc::new).collect(); + targets = new_targets + .into_iter() + .map(|t| ScheduledTarget::new(Arc::new(t))) + .collect(); } Some(ScrapeCommand::Shutdown) | None => return, } @@ -803,7 +892,7 @@ impl ScrapeManager { max_in_flight: usize, exec_one: Exec, ) where - Exec: Fn(Arc) -> Fut, + Exec: Fn(ScheduledTarget) -> Fut, Fut: Future>, { let start_now = Instant::now(); @@ -822,7 +911,7 @@ impl ScrapeManager { while in_flight.len() < max_in_flight && let Some(due) = schedule.pop_due(now) { - let fut = exec_one(due.target); + let fut = exec_one(due.scheduled); in_flight.push(async move { let out = fut.await; (due.slot_id, due.generation, out) diff --git a/crates/scrape-manager/src/scrape/schedule.rs b/crates/scrape-manager/src/scrape/schedule.rs index e67b033..f173bca 100644 --- a/crates/scrape-manager/src/scrape/schedule.rs +++ b/crates/scrape-manager/src/scrape/schedule.rs @@ -14,8 +14,12 @@ use std::cmp::{Ordering, Reverse}; use std::collections::{BinaryHeap, HashMap}; use std::sync::Arc; +#[cfg(feature = "arrow")] +use std::sync::OnceLock; use std::time::Duration; +#[cfg(feature = "arrow")] +use prometheus_expfmt::arrow::TargetAttributes; use tokio::time::Instant; use crate::scrape::ScrapeTarget; @@ -67,9 +71,38 @@ fn fnv1a_bytes(mut hash: u64, bytes: &[u8]) -> u64 { hash } +/// Scheduled target handle keeping lazily-built [`TargetAttributes`] outside [`ScrapeTarget`] so +/// [`ScrapeTarget`] remains pure immutable data usable as a `HashMap`/`HashSet` key. +#[derive(Clone, Debug)] +pub(crate) struct ScheduledTarget { + pub(crate) target: Arc, + #[cfg(feature = "arrow")] + target_attrs: Arc>>>, +} + +impl ScheduledTarget { + pub(crate) fn new(target: Arc) -> Self { + Self { + target, + #[cfg(feature = "arrow")] + target_attrs: Arc::new(OnceLock::new()), + } + } + + #[cfg(feature = "arrow")] + pub(crate) fn target_attributes(&self) -> Option> { + if self.target.labels().is_empty() { + return None; + } + self.target_attrs + .get_or_init(|| self.target.to_target_attributes().map(Arc::new)) + .clone() + } +} + #[derive(Debug)] struct TargetSlot { - target: Arc, + scheduled: ScheduledTarget, generation: u64, next_due: Instant, seen_epoch: u64, @@ -104,7 +137,7 @@ impl PartialOrd for HeapEntry { pub(crate) struct DueTarget { pub(crate) slot_id: usize, pub(crate) generation: u64, - pub(crate) target: Arc, + pub(crate) scheduled: ScheduledTarget, } /// Single-structure min-heap target timer queue (`O(N)` memory, `O(log N)` per scrape tick). @@ -170,7 +203,7 @@ impl TargetSchedule { slot.seen_epoch = epoch; let due = slot.next_due; let generation = slot.generation; - let target_arc = Arc::clone(&slot.target); + let target_arc = Arc::clone(&slot.scheduled.target); self.index_by_target.insert(target_arc, slot_id); self.heap.push(Reverse(HeapEntry { due, @@ -187,7 +220,7 @@ impl TargetSchedule { let target_arc = Arc::new(target); let slot = TargetSlot { - target: Arc::clone(&target_arc), + scheduled: ScheduledTarget::new(Arc::clone(&target_arc)), generation, next_due: due, seen_epoch: epoch, @@ -311,7 +344,7 @@ impl TargetSchedule { return Some(DueTarget { slot_id: entry.slot_id, generation: slot.generation, - target: Arc::clone(&slot.target), + scheduled: slot.scheduled.clone(), }); } None @@ -328,7 +361,7 @@ impl TargetSchedule { { slot.in_flight = false; if slot.removed { - Some(Arc::clone(&slot.target)) + Some(Arc::clone(&slot.scheduled.target)) } else { if slot.next_due <= now { let (next_due, missed_ticks) = @@ -428,14 +461,14 @@ mod tests { let due_at_first = first_due; let popped = sched.pop_due(due_at_first).expect("t1 due"); - assert_eq!(popped.target.url(), t1.url()); + assert_eq!(popped.scheduled.target.url(), t1.url()); assert_eq!(sched.take_overruns(), 0); // Leave t1 `in_flight` across its next due tick (`first_due + 10s`) -> skips and records 1 overrun. let second_tick = first_due + interval; while let Some(other) = sched.pop_due(second_tick) { // Only t2 can pop while t1 is still in_flight. - assert_eq!(other.target.url(), t2.url()); + assert_eq!(other.scheduled.target.url(), t2.url()); sched.complete(other.slot_id, other.generation, second_tick); } assert_eq!(sched.take_overruns(), 1); @@ -449,7 +482,7 @@ mod tests { let third_tick = first_due + interval * 2; let mut fired_urls = Vec::new(); while let Some(d) = sched.pop_due(third_tick) { - fired_urls.push(d.target.url().as_str().to_owned()); + fired_urls.push(d.scheduled.target.url().as_str().to_owned()); sched.complete(d.slot_id, d.generation, third_tick); } assert!(fired_urls.iter().any(|u| u == t1.url().as_str())); @@ -489,7 +522,7 @@ mod tests { let next = sched .pop_due(first_due + interval * 2) .expect("fires after completion"); - assert_eq!(next.target.url(), t1.url()); + assert_eq!(next.scheduled.target.url(), t1.url()); } #[tokio::test(start_paused = true)] @@ -515,7 +548,7 @@ mod tests { let resumed = sched .pop_due(first_due + interval * 2) .expect("fires on next grid tick"); - assert_eq!(resumed.target.url(), t1.url()); + assert_eq!(resumed.scheduled.target.url(), t1.url()); } #[tokio::test(start_paused = true)] From 3673bd5552d768a17b43e28e3b271060668ca244 Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Wed, 18 Nov 2026 05:41:44 +0000 Subject: [PATCH 2/2] scrape-manager: add discovery relabeling, honor_labels, and metric_relabel_configs tests and benchmark Add unit tests for ScrapeConfig::populate_targets, Target.URL() query parameter merging, IPv6/scheme/address validation, and relabel config validation. Add an end-to-end HTTP integration test exercising discovery relabel_configs, target ResourceAttrs emission, honor_labels = false and true, exemplar attribute preservation, Stage 1 and Stage 2 metric_relabel_configs, empty post-relabel scrapes, and MissingMetricName failure reporting. Add a relabeling scrape benchmark case and update specs/SPEC.md. Signed-off-by: Bartek Plotka --- Cargo.lock | 1 + crates/scrape-manager/Cargo.toml | 1 + crates/scrape-manager/benches/scrape.rs | 70 ++++ crates/scrape-manager/src/scrape/config.rs | 162 ++++++++ crates/scrape-manager/tests/scrape_manager.rs | 364 ++++++++++++++++++ specs/SPEC.md | 2 +- 6 files changed, 599 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 28ddd7c..5a57df7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,7 @@ dependencies = [ name = "prometheus-scrape-manager" version = "0.1.0" dependencies = [ + "arrow-array", "benchfmt", "flate2", "futures-util", diff --git a/crates/scrape-manager/Cargo.toml b/crates/scrape-manager/Cargo.toml index 18a4268..c29cdd8 100644 --- a/crates/scrape-manager/Cargo.toml +++ b/crates/scrape-manager/Cargo.toml @@ -32,6 +32,7 @@ thiserror = { workspace = true } tokio = { workspace = true } [dev-dependencies] +arrow-array = { workspace = true } benchfmt = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "test-util"] } diff --git a/crates/scrape-manager/benches/scrape.rs b/crates/scrape-manager/benches/scrape.rs index 75e72c5..a9a7482 100644 --- a/crates/scrape-manager/benches/scrape.rs +++ b/crates/scrape-manager/benches/scrape.rs @@ -31,6 +31,7 @@ use std::time::Duration; use benchfmt::{CountingAlloc, Suite}; use prometheus_expfmt::arrow::OtapMetricRecords; use prometheus_expfmt::format::Format; +use prometheus_relabel::{Action, Config as RelabelConfig, METRIC_NAME_LABEL, Regexp}; use prometheus_scrape_manager::scrape::{ ArrowScrapeSink, BodyBufferPool, FetchedScrape, SchedulingMode, ScrapeConfig, ScrapeError, ScrapeManager, ScrapeReport, ScrapeTarget, TargetFetcher, @@ -189,5 +190,74 @@ fn main() { } } + // Exercise Stage 1 (`__name__` drop) + Stage 2 (`labeldrop` + single-label value transform) + // `metric_relabel_configs` compiled once inside `ScrapeManager::new`. + { + let payload = Arc::clone(&payload); + let targets = Arc::clone(&targets); + let scheduling = SchedulingMode::concurrent_with_pools( + NonZeroUsize::new(8).unwrap(), + NonZeroUsize::new(2).unwrap(), + NonZeroUsize::new(1).unwrap(), + ); + let metric_relabel_configs = vec![ + RelabelConfig { + source_labels: vec![METRIC_NAME_LABEL.to_owned()], + regex: Regexp::must_new("go_.*"), + action: Action::Drop, + ..RelabelConfig::default() + }, + RelabelConfig { + regex: Regexp::must_new("handler"), + action: Action::LabelDrop, + ..RelabelConfig::default() + }, + RelabelConfig { + source_labels: vec!["code".to_owned()], + regex: Regexp::must_new("([0-9])[0-9]{2}"), + target_label: "code_class".to_owned(), + replacement: "${1}xx".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + ]; + + suite = suite.bench( + "ScrapeCycle/targets=16/header_wait=none/mode=concurrent_k8_b2_d1_relabel", + move |b| { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("current_thread runtime"); + + let fetcher = BenchFetcher { + payload: Arc::clone(&payload), + header_wait: Duration::ZERO, + }; + let cfg = ScrapeConfig::new("bench") + .with_scheduling(scheduling) + .with_max_data_points_per_batch(256) + .with_metric_relabel_configs(metric_relabel_configs.clone()); + let manager = ScrapeManager::new(cfg, fetcher).expect("manager"); + let sink = BlackBoxSink; + + rt.block_on(async { + let _ = manager.run_once_arrow(&targets, &sink).await; + }); + + b.set_bytes(total_bytes); + b.report_allocs(); + while b.keep_running() { + rt.block_on(async { + let out = manager + .run_once_arrow(black_box(targets.as_slice()), &sink) + .await; + let _ = black_box(out); + }); + } + }, + ); + } + suite.run(); } diff --git a/crates/scrape-manager/src/scrape/config.rs b/crates/scrape-manager/src/scrape/config.rs index 8e13611..b08bec0 100644 --- a/crates/scrape-manager/src/scrape/config.rs +++ b/crates/scrape-manager/src/scrape/config.rs @@ -829,6 +829,18 @@ mod tests { .with_scrape_interval(Duration::from_secs(5)) .with_scrape_timeout(Duration::from_secs(6)), ScrapeConfig::new("job").with_user_agent("bad\r\nheader"), + ScrapeConfig::new("job").with_scheme("ftp"), + ScrapeConfig::new("job").with_relabel_configs(vec![RelabelConfig { + action: prometheus_relabel::Action::HashMod, + target_label: "shard".to_owned(), + modulus: 0, + ..RelabelConfig::default() + }]), + ScrapeConfig::new("job").with_metric_relabel_configs(vec![RelabelConfig { + action: prometheus_relabel::Action::Replace, + target_label: String::new(), + ..RelabelConfig::default() + }]), ScrapeConfig::new("job") .with_scheduling(SchedulingMode::concurrent_with_pools(one, two, one)), ScrapeConfig::new("job") @@ -841,4 +853,154 @@ mod tests { ); } } + + #[test] + fn populate_targets_applies_relabel_configs_params_and_url_rules() { + use crate::discovery::StaticConfig; + use prometheus_relabel::{Action, Regexp}; + + let group_a = StaticConfig { + targets: vec![ + "10.0.0.1:9100".to_owned(), + "10.0.0.2:9100".to_owned(), + "[::1]:9443".to_owned(), + ], + labels: BTreeMap::from([ + ("env".to_owned(), "prod".to_owned()), + ("__meta_role".to_owned(), "api".to_owned()), + ]), + } + .into_target_group("0"); + + let cfg = ScrapeConfig::new("api-pool") + .with_metrics_path("/custom") + .with_param("module", vec!["default".to_owned(), "extra".to_owned()]) + .with_relabel_configs(vec![ + // Drop 10.0.0.2. + RelabelConfig { + source_labels: vec![ADDRESS_LABEL.to_owned()], + regex: Regexp::must_new("10\\.0\\.0\\.2:.*"), + action: Action::Drop, + ..RelabelConfig::default() + }, + // Override the first `module` query parameter while keeping secondary values. + RelabelConfig { + source_labels: vec!["__meta_role".to_owned()], + target_label: "__param_module".to_owned(), + replacement: "http_$1".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + // Add a new `__param_target` query parameter and promote `__meta_role` to `role`. + RelabelConfig { + source_labels: vec![ADDRESS_LABEL.to_owned()], + target_label: "__param_target".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + RelabelConfig { + source_labels: vec!["__meta_role".to_owned()], + target_label: "role".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + // Switch IPv6 target to https and `/probe`. + RelabelConfig { + source_labels: vec![ADDRESS_LABEL.to_owned()], + regex: Regexp::must_new("\\[::1\\]:.*"), + target_label: SCHEME_LABEL.to_owned(), + replacement: "https".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + RelabelConfig { + source_labels: vec![ADDRESS_LABEL.to_owned()], + regex: Regexp::must_new("\\[::1\\]:.*"), + target_label: METRICS_PATH_LABEL.to_owned(), + replacement: "/probe".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + ]); + + let targets = cfg.populate_targets(&[group_a]).expect("populate_targets"); + assert_eq!(targets.len(), 2); + + assert_eq!( + targets[0].url().as_str(), + "http://10.0.0.1:9100/custom?module=http_api&module=extra&target=10.0.0.1%3A9100" + ); + assert_eq!( + targets[0].labels(), + &BTreeMap::from([ + ("env".to_owned(), "prod".to_owned()), + ("instance".to_owned(), "10.0.0.1:9100".to_owned()), + ("job".to_owned(), "api-pool".to_owned()), + ("role".to_owned(), "api".to_owned()), + ]) + ); + + assert_eq!( + targets[1].url().as_str(), + "https://[::1]:9443/probe?module=http_api&module=extra&target=%5B%3A%3A1%5D%3A9443" + ); + assert_eq!( + targets[1].labels().get("instance").map(String::as_str), + Some("[::1]:9443") + ); + + #[cfg(feature = "arrow")] + { + let ta = targets[0] + .to_target_attributes() + .expect("non-empty target attributes"); + assert_eq!( + ta.resource_attributes() + .get("service.name") + .map(String::as_str), + Some("api-pool") + ); + assert_eq!( + ta.resource_attributes() + .get("service.instance.id") + .map(String::as_str), + Some("10.0.0.1:9100") + ); + assert_eq!( + ta.resource_attributes().get("env").map(String::as_str), + Some("prod") + ); + assert_eq!( + ta.resource_attributes().get("role").map(String::as_str), + Some("api") + ); + } + + // Relabeling that clears `__address__` or injects URL userinfo/query fails cleanly. + let bad_addr_group = StaticConfig { + targets: vec!["user:pass@127.0.0.1:9100".to_owned()], + labels: BTreeMap::new(), + } + .into_target_group("bad"); + assert!(matches!( + ScrapeConfig::new("bad").populate_targets(&[bad_addr_group]), + Err(ScrapeError::InvalidConfig(_)) + )); + + let strip_addr = ScrapeConfig::new("strip").with_relabel_configs(vec![RelabelConfig { + target_label: ADDRESS_LABEL.to_owned(), + replacement: String::new(), + action: Action::Replace, + ..RelabelConfig::default() + }]); + let g = StaticConfig { + targets: vec!["127.0.0.1:9100".to_owned()], + labels: BTreeMap::new(), + } + .into_target_group("0"); + assert!(matches!( + strip_addr.populate_targets(&[g]), + Err(ScrapeError::InvalidConfig(_)) + )); + } } diff --git a/crates/scrape-manager/tests/scrape_manager.rs b/crates/scrape-manager/tests/scrape_manager.rs index a4aa656..6f126a8 100644 --- a/crates/scrape-manager/tests/scrape_manager.rs +++ b/crates/scrape-manager/tests/scrape_manager.rs @@ -750,3 +750,367 @@ async fn cooperative_arrow_decode_reports_partial_errors_reclaims_pools_on_break assert_eq!(stats.scrapes_succeeded, 2); assert_eq!(stats.scrapes_failed, 2); } + +#[cfg(feature = "arrow")] +#[tokio::test] +async fn end_to_end_discovery_relabel_honor_labels_and_metric_relabeling_over_http() { + use std::collections::BTreeMap; + + use arrow_array::types::{UInt8Type, UInt16Type}; + use arrow_array::{Array, DictionaryArray, RecordBatch, StringArray}; + use prometheus_expfmt::arrow::OtapMetricRecords; + use prometheus_relabel::{Action, Config as RelabelConfig, METRIC_NAME_LABEL, Regexp}; + use prometheus_scrape_manager::discovery::{StaticConfig, resolve_static_configs}; + use prometheus_scrape_manager::scrape::{ArrowScrapeSink, ScrapeReport}; + + fn read_str_col(batch: &RecordBatch, col_name: &str) -> Vec { + let col = batch.column_by_name(col_name).expect("column exists"); + if let Some(arr) = col.as_any().downcast_ref::() { + return (0..arr.len()).map(|i| arr.value(i).to_owned()).collect(); + } + if let Some(dict) = col.as_any().downcast_ref::>() { + let vals = dict + .values() + .as_any() + .downcast_ref::() + .expect("utf8 dict values"); + return (0..dict.len()) + .map(|i| vals.value(usize::from(dict.keys().value(i))).to_owned()) + .collect(); + } + if let Some(dict) = col.as_any().downcast_ref::>() { + let vals = dict + .values() + .as_any() + .downcast_ref::() + .expect("utf8 dict values"); + return (0..dict.len()) + .map(|i| vals.value(usize::from(dict.keys().value(i))).to_owned()) + .collect(); + } + panic!("unexpected string column encoding for {col_name}"); + } + + fn read_kv_map(batch: &RecordBatch) -> BTreeMap { + let keys = read_str_col(batch, "key"); + let vals = read_str_col(batch, "str"); + keys.into_iter().zip(vals).collect() + } + + struct CaptureSink { + batches: Mutex>, + reports: Mutex>, + stop_after_reports: Option, + } + + impl CaptureSink { + fn new(stop_after_reports: Option) -> Self { + Self { + batches: Mutex::new(Vec::new()), + reports: Mutex::new(Vec::new()), + stop_after_reports, + } + } + } + + impl ArrowScrapeSink for CaptureSink { + async fn on_batch( + &self, + _target: &ScrapeTarget, + _scrape_timestamp_ms: i64, + records: OtapMetricRecords, + ) -> ControlFlow<()> { + self.batches.lock().unwrap().push(records); + ControlFlow::Continue(()) + } + + async fn on_scrape_complete( + &self, + _target: &ScrapeTarget, + report: ScrapeReport, + ) -> ControlFlow<()> { + let mut reports = self.reports.lock().unwrap(); + reports.push(report); + if self + .stop_after_reports + .is_some_and(|limit| reports.len() >= limit) + { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + } + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("addr"); + + let received_reqs = Arc::new(Mutex::new(Vec::::new())); + let received_reqs_srv = Arc::clone(&received_reqs); + + let payload = concat!( + "# TYPE go_goroutines gauge\n", + "go_goroutines 42\n", + "# TYPE http_requests counter\n", + "http_requests_total{job=\"inner\",method=\"get\",tmp_debug=\"dropme\"} 7 # {trace_id=\"abc\",job=\"ex_job\"} 1.0\n", + "# EOF\n", + ); + + // Expect 5 HTTP scrapes: honor_labels=false (scrape_once), honor_labels=true (scrape_once), + // continuous run_arrow (1 scrape), drop-all metric relabel, and strip-__name__ error. + let server = tokio::spawn(async move { + for _ in 0..5 { + let (mut sock, _) = listener.accept().await.expect("accept"); + let mut buf = [0u8; 1024]; + let n = sock.read(&mut buf).await.expect("read"); + let first_line = String::from_utf8_lossy(&buf[..n]) + .lines() + .next() + .unwrap_or("") + .to_owned(); + received_reqs_srv.lock().unwrap().push(first_line); + + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/openmetrics-text; version=1.0.0; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + payload.len(), + payload + ); + sock.write_all(resp.as_bytes()).await.expect("write"); + } + }); + + let groups = resolve_static_configs(&[StaticConfig { + targets: vec![addr.to_string(), "127.0.0.1:1".to_owned()], + labels: BTreeMap::from([ + ("cluster".to_owned(), "eu-west".to_owned()), + ("__meta_probe_target".to_owned(), "api".to_owned()), + ]), + }]); + + let relabel_configs = vec![ + // Drop the unreachable dummy target before scraping. + RelabelConfig { + source_labels: vec!["__address__".to_owned()], + regex: Regexp::must_new("127\\.0\\.0\\.1:1"), + action: Action::Drop, + ..RelabelConfig::default() + }, + RelabelConfig { + target_label: "instance".to_owned(), + replacement: "api-1".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + RelabelConfig { + source_labels: vec!["__meta_probe_target".to_owned()], + target_label: "__param_target".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + RelabelConfig { + target_label: "__param_module".to_owned(), + replacement: "http_2xx".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + ]; + + let metric_relabel_configs = vec![ + // Stage 1: drop `go_.*` before parsing. + RelabelConfig { + source_labels: vec![METRIC_NAME_LABEL.to_owned()], + regex: Regexp::must_new("go_.*"), + action: Action::Drop, + ..RelabelConfig::default() + }, + // Stage 2: drop `tmp_.*` labels. + RelabelConfig { + regex: Regexp::must_new("tmp_.*"), + action: Action::LabelDrop, + ..RelabelConfig::default() + }, + // Stage 2: uppercase `method`. + RelabelConfig { + source_labels: vec!["method".to_owned()], + target_label: "method".to_owned(), + action: Action::Uppercase, + ..RelabelConfig::default() + }, + // Stage 2: read target label `cluster` into per-point `src_cluster`. + RelabelConfig { + source_labels: vec!["cluster".to_owned()], + target_label: "src_cluster".to_owned(), + action: Action::Replace, + ..RelabelConfig::default() + }, + ]; + + let base_cfg = ScrapeConfig::new("api-job") + .with_scrape_interval(Duration::from_secs(5)) + .with_scrape_timeout(Duration::from_secs(3)) + .with_metrics_path("/probe") + .with_param("module", vec!["default".to_owned(), "extra".to_owned()]) + .with_relabel_configs(relabel_configs) + .with_metric_relabel_configs(metric_relabel_configs); + + let targets = base_cfg + .populate_targets(&groups) + .expect("populate targets"); + assert_eq!(targets.len(), 1); + + // 1. honor_labels = false (default): exposed `job="inner"` becomes `exported_job="inner"`, + // while exemplar `job="ex_job"` stays intact and ResourceAttrs carries target metadata. + let mgr_default = ScrapeManager::with_reqwest(base_cfg.clone()).expect("mgr default"); + let sink_default = CaptureSink::new(None); + let rep = match mgr_default + .scrape_once_arrow(&targets[0], &sink_default) + .await + { + ControlFlow::Continue(r) => r, + ControlFlow::Break(()) => panic!("unexpected break"), + }; + assert!(rep.up, "expected up report: {rep:?}"); + assert_eq!(rep.data_points_decoded, 1); + assert_eq!(rep.batches_emitted, 1); + + { + let batches = sink_default.batches.lock().unwrap(); + let batch = &batches[0]; + assert_eq!( + read_str_col(batch.univariate_metrics().unwrap(), "name"), + vec!["http_requests_total"] + ); + assert_eq!( + read_kv_map(batch.resource_attrs().unwrap()), + BTreeMap::from([ + ("cluster".to_owned(), "eu-west".to_owned()), + ("service.instance.id".to_owned(), "api-1".to_owned()), + ("service.name".to_owned(), "api-job".to_owned()), + ]) + ); + assert_eq!( + read_kv_map(batch.number_dp_attrs().unwrap()), + BTreeMap::from([ + ("exported_job".to_owned(), "inner".to_owned()), + ("method".to_owned(), "GET".to_owned()), + ("src_cluster".to_owned(), "eu-west".to_owned()), + ]) + ); + assert_eq!( + read_kv_map(batch.number_dp_exemplar_attrs().unwrap()), + BTreeMap::from([ + ("job".to_owned(), "ex_job".to_owned()), + ("trace_id".to_owned(), "abc".to_owned()), + ]) + ); + } + + // 2. honor_labels = true: exposed `job="inner"` is preserved without `exported_` renaming. + let mgr_honor = + ScrapeManager::with_reqwest(base_cfg.clone().with_honor_labels(true)).expect("mgr honor"); + let sink_honor = CaptureSink::new(None); + let rep_honor = match mgr_honor.scrape_once_arrow(&targets[0], &sink_honor).await { + ControlFlow::Continue(r) => r, + ControlFlow::Break(()) => panic!("unexpected break"), + }; + assert!(rep_honor.up); + { + let batches = sink_honor.batches.lock().unwrap(); + let batch_honor = &batches[0]; + assert_eq!( + read_kv_map(batch_honor.number_dp_attrs().unwrap()), + BTreeMap::from([ + ("job".to_owned(), "inner".to_owned()), + ("method".to_owned(), "GET".to_owned()), + ("src_cluster".to_owned(), "eu-west".to_owned()), + ]) + ); + } + + // 3. Continuous `run_arrow` (via ScheduledTarget / TargetSchedule) emits the same ResourceAttrs. + let mgr_loop = ScrapeManager::with_reqwest( + base_cfg + .clone() + .with_scrape_interval(Duration::from_millis(50)) + .with_scrape_timeout(Duration::from_millis(50)) + .with_scheduling(SchedulingMode::sequential()), + ) + .expect("mgr loop"); + let sink_loop = CaptureSink::new(Some(1)); + let (_ctrl, ctrl_rx) = ScrapeManager::control_channel(); + mgr_loop + .run_arrow(targets.clone(), ctrl_rx, &sink_loop) + .await; + assert_eq!(sink_loop.reports.lock().unwrap().len(), 1); + assert_eq!( + read_kv_map( + sink_loop.batches.lock().unwrap()[0] + .resource_attrs() + .unwrap() + ), + BTreeMap::from([ + ("cluster".to_owned(), "eu-west".to_owned()), + ("service.instance.id".to_owned(), "api-1".to_owned()), + ("service.name".to_owned(), "api-job".to_owned()), + ]) + ); + + // 4. Metric relabeling dropping every series still succeeds (`up == true`, 0 batches). + let mgr_drop_all = ScrapeManager::with_reqwest( + ScrapeConfig::new("drop-all").with_metric_relabel_configs(vec![RelabelConfig { + source_labels: vec![METRIC_NAME_LABEL.to_owned()], + regex: Regexp::must_new(".*"), + action: Action::Drop, + ..RelabelConfig::default() + }]), + ) + .expect("mgr drop all"); + let sink_drop = CaptureSink::new(None); + let rep_drop = match mgr_drop_all + .scrape_once_arrow(&targets[0], &sink_drop) + .await + { + ControlFlow::Continue(r) => r, + ControlFlow::Break(()) => panic!("unexpected break"), + }; + assert!(rep_drop.up); + assert_eq!(rep_drop.data_points_decoded, 0); + assert_eq!(rep_drop.batches_emitted, 0); + + // 5. Dropping `__name__` while other labels remain fails the scrape (`up == false`). + let mgr_strip_name = ScrapeManager::with_reqwest( + ScrapeConfig::new("strip-name").with_metric_relabel_configs(vec![RelabelConfig { + target_label: METRIC_NAME_LABEL.to_owned(), + replacement: String::new(), + action: Action::Replace, + ..RelabelConfig::default() + }]), + ) + .expect("mgr strip name"); + let sink_strip = CaptureSink::new(None); + let rep_strip = match mgr_strip_name + .scrape_once_arrow(&targets[0], &sink_strip) + .await + { + ControlFlow::Continue(r) => r, + ControlFlow::Break(()) => panic!("unexpected break"), + }; + assert!(!rep_strip.up); + assert!( + rep_strip + .error + .as_deref() + .is_some_and(|e| e.contains("missing metric name")), + "expected MissingMetricName error, got {:?}", + rep_strip.error + ); + + server.await.expect("server join"); + let reqs = received_reqs.lock().unwrap(); + assert_eq!(reqs.len(), 5); + assert!( + reqs.iter() + .all(|line| line == "GET /probe?module=http_2xx&module=extra&target=api HTTP/1.1"), + "unexpected HTTP request lines: {reqs:?}" + ); +} diff --git a/specs/SPEC.md b/specs/SPEC.md index dd3106f..5fbd9c7 100644 --- a/specs/SPEC.md +++ b/specs/SPEC.md @@ -175,7 +175,7 @@ pub fn arrow_to_metric_families( - `SchedulingMode::Concurrent` separates three limits: `max_in_flight` bounds simultaneous HTTP requests waiting on TCP connect and response headers, `max_buffered_bodies` bounds reusable `Vec` response buffers in `BodyBufferPool`, and `max_decoders` bounds reusable `ArrowDecoder` instances in `DecoderPool`. A slow target waiting on server-side `/metrics` generation holds zero payload buffers and zero decoders until HTTP `200 OK` headers pass `Content-Length <= max_body_bytes`. Any buffer that grows past `max_retained_body_bytes` on an outlier target shrinks back to `max_retained_body_bytes` when returned to `BodyBufferPool`, and any `ArrowDecoder` whose scratch capacity exceeds `max_retained_decoder_bytes` resets on return to `DecoderPool`. - `SchedulingMode::Sequential` remains available as a first-class option for callers that want strict one-at-a-time scraping with `1` body buffer and `1` decoder, using either a job-wide cycle tick via `SequentialSchedule::CycleTick` or per-target FNV-1a offsets via `SequentialSchedule::Jittered`. - `ReqwestFetcher` sends `Accept`, `Accept-Encoding: gzip`, `User-Agent`, and `X-Prometheus-Scrape-Timeout-Seconds` headers matching [`targetScraper.scrape`](https://github.com/prometheus/prometheus/blob/v0.315.1/scrape/scrape.go#L824-L836), and streams compressed or plain chunks with a hard `max_body_bytes` cap on uncompressed output so oversized responses abort mid-stream. -- `ScrapeManager` drives concurrent scrapes via `FuturesUnordered` without `tokio::spawn`, keeping `ArrowScrapeSink` and `RawScrapeSink` free of `Send` bounds for thread-per-core `LocalSet` receivers in `open-telemetry/otel-arrow`. Chunks delivered to `ArrowScrapeSink::on_batch` are tentative until `ArrowScrapeSink::on_scrape_complete` observes `report.up == true`; if decoding fails mid-stream, `ScrapeReport` records `up == false` alongside the partial `batches_emitted` count so transactional sinks can discard staged chunks like Go `app.Rollback()`. +- `ScrapeManager` drives concurrent scrapes via `FuturesUnordered` without `tokio::spawn`, keeping `ArrowScrapeSink` and `RawScrapeSink` free of `Send` bounds for thread-per-core `LocalSet` receivers in `open-telemetry/otel-arrow`. `ScrapeConfig::populate_targets` executes target discovery `relabel_configs` (`populate_labels`) and merges `params` / `__param_*` query parameters into `ScrapeTarget` endpoints matching Prometheus `Target.URL()`, while `ScrapeManager` pre-compiles `metric_relabel_configs` once into a shared `Arc` and attaches cached per-target `TargetAttributes` and `honor_labels` during Arrow decoding. Chunks delivered to `ArrowScrapeSink::on_batch` are tentative until `ArrowScrapeSink::on_scrape_complete` observes `report.up == true`; if decoding fails mid-stream, `ScrapeReport` records `up == false` alongside the partial `batches_emitted` count so transactional sinks can discard staged chunks like Go `app.Rollback()`. ## Testing and verification