From 5282bcdfa860daf0a0a56ff301e6784900cd23e4 Mon Sep 17 00:00:00 2001 From: Ethan Olchik Date: Mon, 17 Aug 2026 15:27:19 +0100 Subject: [PATCH 1/2] feat(metrics): let native histograms recover lost resolution `NativeHistogramBuilder` never passed a minimum reset duration to `prometheus-client`, so when a histogram exceeded `max_buckets` the only available mitigation was reducing its schema by one step. Nothing restored the original schema afterwards, making the resolution loss permanent for the lifetime of the process, per label set, and ratcheting further on every subsequent breach. Expose `min_reset_duration` on the builder and forward it to the wrapped configuration. Once the duration has elapsed, a bucket limit breach resets the histogram and restores its configured schema instead of degrading it. The default of one hour bounds any resolution loss; `Duration::ZERO` disables resets and preserves the previous behaviour. Bounding degradation is what makes finer bucket factors practical: callers no longer have to choose a coarse factor purely to stay clear of the bucket cap. Note that a reset also clears the counts, sum, and classic buckets, which is visible as a counter reset in `_bucket`, `_sum`, and `_count` series. --- .../src/metrics/native_histogram.rs | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/foundations-metrics/src/metrics/native_histogram.rs b/foundations-metrics/src/metrics/native_histogram.rs index 4dbf5824..ed5af6ca 100644 --- a/foundations-metrics/src/metrics/native_histogram.rs +++ b/foundations-metrics/src/metrics/native_histogram.rs @@ -1,4 +1,4 @@ -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use foundations_metrics_registry::proto::{self, Bucket, BucketSpan, LabelPair, MetricType}; use prometheus_client::encoding::prometheus_protobuf::{ @@ -242,17 +242,32 @@ pub struct NativeHistogramBuilder { /// /// `0` leaves the bucket count unbounded. pub max_buckets: usize, + + /// How long a degraded histogram must run before the bucket limit is + /// enforced by resetting it instead of permanently reducing its resolution. + /// + /// When [`Self::max_buckets`] is exceeded and at least this long has elapsed + /// since the histogram was created or last reset, the histogram is reset: + /// its schema returns to the value implied by [`Self::bucket_factor`], and + /// its counts, sum, and classic buckets are cleared. Otherwise the schema is + /// reduced by one step, which is not recovered until a later reset. + /// + /// [`Duration::ZERO`] disables resets, so resolution loss becomes permanent + /// for the lifetime of the histogram. + pub min_reset_duration: Duration, } impl NativeHistogramBuilder { /// Creates a builder with the given bucket growth `factor`, the default zero - /// threshold, and an unbounded number of buckets. + /// threshold, an unbounded number of buckets, and a one hour minimum reset + /// duration. pub fn new(factor: f64) -> Self { Self { classic_buckets: None, bucket_factor: factor, zero_threshold: 0.0, max_buckets: 0, + min_reset_duration: Duration::from_secs(3600), } } @@ -284,6 +299,15 @@ impl NativeHistogramBuilder { self } + /// Sets how long a degraded histogram must run before the bucket limit is + /// enforced by resetting it rather than reducing its resolution. + /// + /// [`Duration::ZERO`] disables resets, making resolution loss permanent. + pub fn with_min_reset_duration(mut self, min_reset_duration: Duration) -> Self { + self.min_reset_duration = min_reset_duration; + self + } + /// Translates this builder into the wrapped crate's configuration. /// /// # Panics @@ -309,6 +333,7 @@ impl NativeHistogramBuilder { NativeHistogramConfig::new(self.bucket_factor) .zero_threshold(self.zero_threshold) .max_buckets(self.max_buckets) + .min_reset_duration(self.min_reset_duration) } } @@ -804,4 +829,46 @@ mod tests { assert_eq!(encoded.sample_count, Some(1)); assert_eq!(encoded.zero_threshold, Some(0.001)); } + + #[test] + fn elapsed_min_reset_duration_resets_instead_of_degrading_resolution() { + let histogram: NativeHistogram = NativeHistogramBuilder::new(1.1) + .with_max_buckets(1) + .with_min_reset_duration(Duration::from_nanos(1)) + .new_metric(); + + let schema_before = encoded_histogram(&histogram.encode_metric_value()) + .schema + .expect("schema is present"); + + histogram.observe(1.0); + histogram.observe(1.5); + + let families = histogram.encode_metric_value(); + let encoded = encoded_histogram(&families); + + assert_eq!(encoded.schema, Some(schema_before)); + assert_eq!(encoded.sample_count, Some(1)); + } + + #[test] + fn zero_min_reset_duration_degrades_resolution() { + let histogram: NativeHistogram = NativeHistogramBuilder::new(1.1) + .with_max_buckets(1) + .with_min_reset_duration(Duration::ZERO) + .new_metric(); + + let schema_before = encoded_histogram(&histogram.encode_metric_value()) + .schema + .expect("schema is present"); + + histogram.observe(1.0); + histogram.observe(1.5); + + let families = histogram.encode_metric_value(); + let encoded = encoded_histogram(&families); + + assert!(encoded.schema.expect("schema is present") < schema_before); + assert_eq!(encoded.sample_count, Some(2)); + } } From fa9996814082877a7c199f37ad77440caa7c17fb Mon Sep 17 00:00:00 2001 From: Ethan Olchik Date: Tue, 18 Aug 2026 11:57:32 +0100 Subject: [PATCH 2/2] feat(metrics): expose max_zero_threshold on NativeHistogramBuilder --- foundations-macros/src/metrics/mod.rs | 2 + .../src/metrics/native_histogram.rs | 68 +++++++++++++++++-- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/foundations-macros/src/metrics/mod.rs b/foundations-macros/src/metrics/mod.rs index e28d7340..e5b8c371 100644 --- a/foundations-macros/src/metrics/mod.rs +++ b/foundations-macros/src/metrics/mod.rs @@ -1092,6 +1092,8 @@ mod tests { bucket_factor: 1.1, zero_threshold: 0.0, max_buckets: 0, + min_reset_duration: ::std::time::Duration::from_secs(3600), + max_zero_threshold: 0.0, } }; diff --git a/foundations-metrics/src/metrics/native_histogram.rs b/foundations-metrics/src/metrics/native_histogram.rs index ed5af6ca..c503dc56 100644 --- a/foundations-metrics/src/metrics/native_histogram.rs +++ b/foundations-metrics/src/metrics/native_histogram.rs @@ -249,18 +249,29 @@ pub struct NativeHistogramBuilder { /// When [`Self::max_buckets`] is exceeded and at least this long has elapsed /// since the histogram was created or last reset, the histogram is reset: /// its schema returns to the value implied by [`Self::bucket_factor`], and - /// its counts, sum, and classic buckets are cleared. Otherwise the schema is - /// reduced by one step, which is not recovered until a later reset. + /// its counts, sum, and classic buckets are cleared. Otherwise the zero + /// threshold may be widened up to [`Self::max_zero_threshold`], and if that + /// is not enough the schema is reduced by one step, which is not recovered + /// until a later reset. /// /// [`Duration::ZERO`] disables resets, so resolution loss becomes permanent /// for the lifetime of the histogram. pub min_reset_duration: Duration, + + /// Upper bound on how far the zero bucket may be widened while enforcing + /// [`Self::max_buckets`]. + /// + /// When the bucket limit is exceeded and a reset is not yet due, the + /// histogram first tries to absorb sparse buckets near zero into a wider + /// zero bucket. Widening stops at this value. `0.0` (the default) disables + /// widening, so the next step is schema reduction. + pub max_zero_threshold: f64, } impl NativeHistogramBuilder { /// Creates a builder with the given bucket growth `factor`, the default zero - /// threshold, an unbounded number of buckets, and a one hour minimum reset - /// duration. + /// threshold, an unbounded number of buckets, a one hour minimum reset + /// duration, and zero-bucket widening disabled. pub fn new(factor: f64) -> Self { Self { classic_buckets: None, @@ -268,6 +279,7 @@ impl NativeHistogramBuilder { zero_threshold: 0.0, max_buckets: 0, min_reset_duration: Duration::from_secs(3600), + max_zero_threshold: 0.0, } } @@ -308,12 +320,22 @@ impl NativeHistogramBuilder { self } + /// Sets how far the zero bucket may be widened while enforcing the bucket + /// limit. + /// + /// `0.0` disables widening. Must be finite and non-negative. + pub fn with_max_zero_threshold(mut self, max_zero_threshold: f64) -> Self { + self.max_zero_threshold = max_zero_threshold; + self + } + /// Translates this builder into the wrapped crate's configuration. /// /// # Panics /// - /// Panics if `bucket_factor` is not greater than `1.0` or if - /// `zero_threshold` is not finite. + /// Panics if `bucket_factor` is not greater than `1.0`, if + /// `zero_threshold` is not finite, or if `max_zero_threshold` is not finite + /// and non-negative. #[track_caller] fn config(&self) -> NativeHistogramConfig { // Validated here rather than left to `prometheus-client`: its assertions @@ -329,11 +351,17 @@ impl NativeHistogramBuilder { "native histogram zero threshold must be finite, but was {}", self.zero_threshold ); + assert!( + self.max_zero_threshold.is_finite() && self.max_zero_threshold >= 0.0, + "native histogram max zero threshold must be finite and non-negative, but was {}", + self.max_zero_threshold + ); NativeHistogramConfig::new(self.bucket_factor) .zero_threshold(self.zero_threshold) .max_buckets(self.max_buckets) .min_reset_duration(self.min_reset_duration) + .max_zero_threshold(self.max_zero_threshold) } } @@ -567,6 +595,7 @@ mod tests { let histogram: NativeHistogram = NativeHistogramBuilder::new(1.5) .with_zero_threshold(0.001) .with_max_buckets(160) + .with_max_zero_threshold(1.0) .new_metric(); histogram.observe(0.5); @@ -871,4 +900,31 @@ mod tests { assert!(encoded.schema.expect("schema is present") < schema_before); assert_eq!(encoded.sample_count, Some(2)); } + + #[test] + fn max_zero_threshold_widens_zero_bucket_before_reducing_resolution() { + // Factor 2^(2^-8) ≈ 1.0027 selects schema 8, matching the upstream + // prometheus-client case that exercises zero-bucket widening. + let histogram: NativeHistogram = NativeHistogramBuilder::new(2f64.powf(2f64.powi(-8))) + .with_max_buckets(1) + .with_min_reset_duration(Duration::ZERO) + .with_max_zero_threshold(1.0) + .new_metric(); + + let schema_before = encoded_histogram(&histogram.encode_metric_value()) + .schema + .expect("schema is present"); + + histogram.observe(2f64.powi(-100)); + histogram.observe(1.0); + + let families = histogram.encode_metric_value(); + let encoded = encoded_histogram(&families); + + assert_eq!(encoded.schema, Some(schema_before)); + assert_eq!(encoded.sample_count, Some(2)); + assert_eq!(encoded.zero_count, Some(1)); + assert!(encoded.zero_threshold.expect("zero threshold") > 0.0); + assert_eq!(encoded.positive_delta.len(), 1); + } }