Skip to content

fix(hll): preserve estimator state across union transformations #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tisonkun merged 6 commits into from
Aug 29, 2026
Merged

fix(hll): preserve estimator state across union transformations #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dc21c61
fix(hll): keep the HIP estimator when a sketch is merged into an empt…
jaideeppyne Aug 28, 2026
42653ea
refactor(hll): preserve estimator state atomically
tisonkun Aug 29, 2026
dc2c260
test(hll): streamline estimator state coverage
tisonkun Aug 29, 2026
0014520
Merge branch 'main' into fix-hll-union-hip
tisonkun Aug 29, 2026
c76f1d2
refactor(hll): generalize estimator naming
tisonkun Aug 29, 2026
80f3b76
refactor(hll): retain estimator visibility
tisonkun Aug 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ All significant changes to this project will be documented in this file.
* T-Digest rejects truncated serialized payloads before allocating, and updating a deserialized digest no longer allows its buffered state to grow without bound.
* Compact HLL4 images now restore all register values correctly.
* `HllSketch::lower_bound` now uses the number of non-zero registers as a floor in HLL mode, matching Java, C++, and Go and avoiding a bound below the distinct count already proven by register hits.
* `HllUnion` now keeps a single HLL-mode input's estimate stable when copying or downsampling it and keeps confidence bounds consistent across HLL4, HLL6, and HLL8 result types, matching Java and C++.
* HLL, Theta, and Tuple deserializers now return `InvalidData` for malformed payload sizes and entry counts instead of risking oversized allocations or decoding failures.
* Malformed CPC images now return `InvalidData` instead of panicking.
* Seeded deserializers now return `InvalidData` rather than panicking when the caller supplies a seed whose hash is the reserved zero value.
Expand Down
37 changes: 15 additions & 22 deletions datasketches/src/hll/array4.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ use crate::common::NumStdDev;
use crate::error::Error;
use crate::hll::Coupon;
use crate::hll::aux_map::AuxMap;
use crate::hll::estimator::HipEstimator;
use crate::hll::estimator::EstimateState;
use crate::hll::estimator::Estimator;
use crate::hll::serialization::COMPACT_FLAG_MASK;
use crate::hll::serialization::COUPON_SIZE_BYTES;
use crate::hll::serialization::CUR_MODE_HLL;
Expand Down Expand Up @@ -70,8 +71,7 @@ pub struct Array4 {
num_at_cur_min: u32,
/// Exception table for values >= 15 after cur_min offset
aux_map: Option<AuxMap>,
/// HIP estimator for cardinality estimation
estimator: HipEstimator,
estimator: Estimator,
}

impl Array4 {
Expand All @@ -84,7 +84,7 @@ impl Array4 {
cur_min: 0,
num_at_cur_min,
aux_map: None,
estimator: HipEstimator::new(lg_config_k),
estimator: Estimator::new(lg_config_k),
}
}

Expand Down Expand Up @@ -125,9 +125,9 @@ impl Array4 {
1 << self.lg_config_k
}

/// Get the current HIP accumulator value
pub(super) fn hip_accum(&self) -> f64 {
self.estimator.hip_accum()
/// Returns the estimate state independently from register-derived cached values.
pub(super) fn estimate_state(&self) -> EstimateState {
self.estimator.estimate_state()
}

/// Set raw 4-bit value in slot
Expand Down Expand Up @@ -270,7 +270,7 @@ impl Array4 {
self.num_at_cur_min = num_at_new;
}

/// Get the current cardinality estimate using HIP estimator
/// Returns the current cardinality estimate.
pub fn estimate(&self) -> f64 {
// Array4 tracks cur_min and num_at_cur_min dynamically
self.estimator
Expand All @@ -297,11 +297,9 @@ impl Array4 {
)
}

/// Set the HIP accumulator value
///
/// This is used when promoting from coupon modes to carry forward the estimate
pub fn set_hip_accum(&mut self, value: f64) {
self.estimator.set_hip_accum(value);
/// Restores estimate state after copying or transforming the same logical sketch.
pub(super) fn restore_estimate_state(&mut self, state: EstimateState) {
self.estimator.restore_estimate_state(state);
}

/// Check if the sketch is empty (all slots are zero)
Expand All @@ -322,7 +320,7 @@ impl Array4 {
let k = 1usize << lg_config_k;
let num_bytes = 1usize << (lg_config_k - 1); // k/2 bytes for 4-bit packing

// Read HIP estimator values from preamble
// Read estimator values from preamble
let hip_accum = cursor
.read_f64_le()
.map_err(insufficient_data("hip_accum"))?;
Expand Down Expand Up @@ -404,12 +402,7 @@ impl Array4 {
aux_map = Some(aux);
}

// Create estimator and restore state
let mut estimator = HipEstimator::new(lg_config_k);
estimator.set_hip_accum(hip_accum);
estimator.set_kxq0(kxq0);
estimator.set_kxq1(kxq1);
estimator.set_out_of_order(ooo);
let estimator = Estimator::from_serialized(hip_accum, kxq0, kxq1, ooo);

Ok(Self {
lg_config_k,
Expand Down Expand Up @@ -449,7 +442,7 @@ impl Array4 {
// COMPACT_FLAG_MASK is always set: aux map entries are written as a compact sequential
// list of populated entries only.
let mut flags = COMPACT_FLAG_MASK;
if self.estimator.is_out_of_order() {
if self.estimator.uses_composite_estimate() {
flags |= OUT_OF_ORDER_FLAG_MASK;
}
bytes.write_u8(flags);
Expand All @@ -460,7 +453,7 @@ impl Array4 {
// Mode byte: HLL mode with HLL4 type
bytes.write_u8(encode_mode_byte(CUR_MODE_HLL, TGT_HLL4));

// Write HIP estimator values
// Write estimator values
bytes.write_f64_le(self.estimator.hip_accum());
bytes.write_f64_le(self.estimator.kxq0());
bytes.write_f64_le(self.estimator.kxq1());
Expand Down
37 changes: 15 additions & 22 deletions datasketches/src/hll/array6.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ use crate::codec::family::Family;
use crate::common::NumStdDev;
use crate::error::Error;
use crate::hll::Coupon;
use crate::hll::estimator::HipEstimator;
use crate::hll::estimator::EstimateState;
use crate::hll::estimator::Estimator;
use crate::hll::serialization::CUR_MODE_HLL;
use crate::hll::serialization::HLL_PREAMBLE_SIZE;
use crate::hll::serialization::HLL_PREINTS;
Expand All @@ -47,8 +48,7 @@ pub struct Array6 {
bytes: Box<[u8]>,
/// Count of slots with value 0
num_zeros: u32,
/// HIP estimator for cardinality estimation
estimator: HipEstimator,
estimator: Estimator,
}

impl Array6 {
Expand All @@ -60,7 +60,7 @@ impl Array6 {
lg_config_k,
bytes: vec![0u8; num_bytes].into_boxed_slice(),
num_zeros: k,
estimator: HipEstimator::new(lg_config_k),
estimator: Estimator::new(lg_config_k),
}
}

Expand Down Expand Up @@ -91,9 +91,9 @@ impl Array6 {
1 << self.lg_config_k
}

/// Get the current HIP accumulator value
pub(super) fn hip_accum(&self) -> f64 {
self.estimator.hip_accum()
/// Returns the estimate state independently from register-derived cached values.
pub(super) fn estimate_state(&self) -> EstimateState {
self.estimator.estimate_state()
}

/// Set value in a slot (6-bit value)
Expand Down Expand Up @@ -145,7 +145,7 @@ impl Array6 {
}
}

/// Get the current cardinality estimate using HIP estimator
/// Returns the current cardinality estimate.
pub fn estimate(&self) -> f64 {
// Array6 doesn't use cur_min (always 0), so num_at_cur_min = num_zeros
self.estimator.estimate(self.lg_config_k, 0, self.num_zeros)
Expand All @@ -163,11 +163,9 @@ impl Array6 {
.lower_bound(self.lg_config_k, 0, self.num_zeros, num_std_dev)
}

/// Set the HIP accumulator value
///
/// This is used when promoting from coupon modes to carry forward the estimate
pub fn set_hip_accum(&mut self, value: f64) {
self.estimator.set_hip_accum(value);
/// Restores estimate state after copying or transforming the same logical sketch.
pub(super) fn restore_estimate_state(&mut self, state: EstimateState) {
self.estimator.restore_estimate_state(state);
}

/// Check if the sketch is empty (all slots are zero)
Expand All @@ -186,7 +184,7 @@ impl Array6 {
let k = 1 << lg_config_k;
let num_bytes = num_bytes_for_k(k);

// Read HIP estimator values from preamble
// Read estimator values from preamble
let hip_accum = cursor
.read_f64_le()
.map_err(insufficient_data("hip_accum"))?;
Expand Down Expand Up @@ -218,12 +216,7 @@ impl Array6 {
.read_exact(&mut data)
.map_err(insufficient_data("data"))?;

// Create estimator and restore state
let mut estimator = HipEstimator::new(lg_config_k);
estimator.set_hip_accum(hip_accum);
estimator.set_kxq0(kxq0);
estimator.set_kxq1(kxq1);
estimator.set_out_of_order(ooo);
let estimator = Estimator::from_serialized(hip_accum, kxq0, kxq1, ooo);

Ok(Self {
lg_config_k,
Expand Down Expand Up @@ -251,7 +244,7 @@ impl Array6 {

// Write flags
let mut flags = 0u8;
if self.estimator.is_out_of_order() {
if self.estimator.uses_composite_estimate() {
flags |= OUT_OF_ORDER_FLAG_MASK;
}
bytes.write_u8(flags);
Expand All @@ -262,7 +255,7 @@ impl Array6 {
// Mode byte: HLL mode with HLL6 type
bytes.write_u8(encode_mode_byte(CUR_MODE_HLL, TGT_HLL6));

// Write HIP estimator values
// Write estimator values
bytes.write_f64_le(self.estimator.hip_accum());
bytes.write_f64_le(self.estimator.kxq0());
bytes.write_f64_le(self.estimator.kxq1());
Expand Down
Loading