fix(theta,tuple): treat equal-theta sketches with no retained entries as identical

CHANGELOG.md

@@ -30,6 +30,7 @@ All significant changes to this project will be documented in this file.
 * Compact HLL4 images now restore all register values correctly.
 * `HllSketch::lower_bound` now uses the number of non-zero registers as a floor in HLL mode, matching Java, C++, and Go and avoiding a bound below the distinct count already proven by register hits.
 * `HllUnion` now keeps a single HLL-mode input's estimate stable when copying or downsampling it and keeps confidence bounds consistent across HLL4, HLL6, and HLL8 result types, matching Java and C++.
+* `ThetaJaccardSimilarity` and `TupleJaccardSimilarity` now report an exact similarity of `1.0` for two non-empty sketches that share a theta and retain no entries, matching Java and C++ and agreeing with `exactly_equal` on the same pair. Such pairs, which arise from a low sampling probability, previously returned the uncertain `{0.0, 0.5, 1.0}` interval, so a sketch was not similar to itself.
 * HLL, Theta, and Tuple deserializers now return `InvalidData` for malformed payload sizes and entry counts instead of risking oversized allocations or decoding failures.
 * Malformed CPC images now return `InvalidData` instead of panicking.
 * Seeded deserializers now return `InvalidData` rather than panicking when the caller supplies a seed whose hash is the reserved zero value.

datasketches/src/thetafamily/common/jaccard_similarity.rs

@@ -187,7 +187,7 @@ where
     let sketch_a_state = (a_num_retained, a_theta);
     let sketch_b_state = (b_num_retained, b_theta);
     let union = compute_union(seed, sketch_a, sketch_b)?;
-    if !union.entries.is_empty() && identical_sets(sketch_a_state, sketch_b_state, &union) {
+    if identical_sets(sketch_a_state, sketch_b_state, &union) {
         return Ok(JaccardSimilarity::exact(1.0));
     }
 

tests-integration/tests/theta_test/jaccard_similarity.rs

@@ -184,7 +184,7 @@ fn test_seed_mismatch() {
 }
 
 #[test]
-fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
+fn test_equal_theta_non_empty_sketches_with_no_retained_entries_are_identical() {
     let mut sketch_a = ThetaSketchBuilder::default()
         .sampling_probability(1e-12)
         .build()
@@ -193,26 +193,56 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
         .sampling_probability(1e-12)
         .build()
         .unwrap();
+    sketch_a.update("apple");
+    sketch_b.update("banana");
+
+    assert!(!sketch_a.is_empty());
+    assert!(!sketch_b.is_empty());
+    assert_eq!(sketch_a.num_retained(), 0);
+    assert_eq!(sketch_b.num_retained(), 0);
+    assert_eq!(sketch_a.theta64(), sketch_b.theta64());
+
+    let operator = ThetaJaccardSimilarity::default();
+    assert_jaccard_exact(operator.compute(&sketch_a, &sketch_b).unwrap(), 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+}
+
+#[test]
+fn test_distinct_theta_non_empty_sketches_with_no_retained_entries_are_uncertain() {
+    let mut sketch_a = ThetaSketchBuilder::default()
+        .sampling_probability(1e-12)
+        .build()
+        .unwrap();
     let mut different_theta = ThetaSketchBuilder::default()
         .sampling_probability(2e-12)
         .build()
         .unwrap();
     sketch_a.update("apple");
-    sketch_b.update("banana");
     different_theta.update("orange");
 
-    assert!(!sketch_a.is_empty());
-    assert!(!sketch_b.is_empty());
     assert_eq!(sketch_a.num_retained(), 0);
-    assert_eq!(sketch_b.num_retained(), 0);
     assert_eq!(different_theta.num_retained(), 0);
 
     let operator = ThetaJaccardSimilarity::default();
-    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
+    let jaccard = operator.compute(&sketch_a, &different_theta).unwrap();
     assert_eq!(jaccard.lower_bound(), 0.0);
     assert_eq!(jaccard.estimate(), 0.5);
     assert_eq!(jaccard.upper_bound(), 1.0);
 
-    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
     assert!(!operator.exactly_equal(&sketch_a, &different_theta).unwrap());
 }
+
+#[test]
+fn test_sketch_is_identical_to_itself_with_no_retained_entries() {
+    let mut sketch = ThetaSketchBuilder::default()
+        .sampling_probability(1e-12)
+        .build()
+        .unwrap();
+    sketch.update("apple");
+
+    assert!(!sketch.is_empty());
+    assert_eq!(sketch.num_retained(), 0);
+
+    let operator = ThetaJaccardSimilarity::default();
+    assert_jaccard_exact(operator.compute(&sketch, &sketch).unwrap(), 1.0);
+}

tests-integration/tests/tuple_test/jaccard_similarity.rs

@@ -134,7 +134,7 @@ fn test_custom_seed_and_seed_mismatch() {
 }
 
 #[test]
-fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
+fn test_equal_theta_non_empty_sketches_with_no_retained_entries_are_identical() {
     let mut sketch_a = default_tuple_sketch_builder()
         .sampling_probability(1e-12)
         .build()
@@ -150,12 +150,49 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
     assert!(!sketch_b.is_empty());
     assert_eq!(sketch_a.num_retained(), 0);
     assert_eq!(sketch_b.num_retained(), 0);
+    assert_eq!(sketch_a.theta64(), sketch_b.theta64());
 
     let operator = TupleJaccardSimilarity::default();
-    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
+    assert_jaccard_exact(operator.compute(&sketch_a, &sketch_b).unwrap(), 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+}
+
+#[test]
+fn test_distinct_theta_non_empty_sketches_with_no_retained_entries_are_uncertain() {
+    let mut sketch_a = default_tuple_sketch_builder()
+        .sampling_probability(1e-12)
+        .build()
+        .unwrap();
+    let mut different_theta = default_tuple_sketch_builder()
+        .sampling_probability(2e-12)
+        .build()
+        .unwrap();
+    sketch_a.update("apple", 1u64);
+    different_theta.update("orange", 1u64);
+
+    assert_eq!(sketch_a.num_retained(), 0);
+    assert_eq!(different_theta.num_retained(), 0);
+
+    let operator = TupleJaccardSimilarity::default();
+    let jaccard = operator.compute(&sketch_a, &different_theta).unwrap();
     assert_eq!(jaccard.lower_bound(), 0.0);
     assert_eq!(jaccard.estimate(), 0.5);
     assert_eq!(jaccard.upper_bound(), 1.0);
 
-    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+    assert!(!operator.exactly_equal(&sketch_a, &different_theta).unwrap());
+}
+
+#[test]
+fn test_sketch_is_identical_to_itself_with_no_retained_entries() {
+    let mut sketch = default_tuple_sketch_builder()
+        .sampling_probability(1e-12)
+        .build()
+        .unwrap();
+    sketch.update("apple", 1u64);
+
+    assert!(!sketch.is_empty());
+    assert_eq!(sketch.num_retained(), 0);
+
+    let operator = TupleJaccardSimilarity::default();
+    assert_jaccard_exact(operator.compute(&sketch, &sketch).unwrap(), 1.0);
 }