feat: add exact equality for Jaccard sketches

datasketches/src/thetafamily/common/jaccard_similarity.rs

@@ -25,6 +25,7 @@ use crate::thetacommon::binomial_bounds;
 use crate::thetacommon::constants::MAX_LG_K;
 use crate::thetacommon::constants::MAX_THETA;
 use crate::thetacommon::constants::MIN_LG_K;
+use crate::thetacommon::hash_table::CompactSketchParts;
 use crate::thetacommon::intersection::IntersectionMergePolicy;
 use crate::thetacommon::intersection::IntersectionState;
 use crate::thetacommon::union::UnionMergePolicy;
@@ -242,44 +243,13 @@ impl JaccardSimilarityOperator {
             return Ok(JaccardSimilarity::exact(0.0));
         }
 
-        let seed_hash = compute_seed_hash(self.seed);
-        if seed_hash != sketch_a.seed_hash() {
-            return Err(Error::invalid_argument(format!(
-                "incompatible seed hash: expected {}, got {}",
-                seed_hash,
-                sketch_a.seed_hash(),
-            )));
-        }
-        if seed_hash != sketch_b.seed_hash() {
-            return Err(Error::invalid_argument(format!(
-                "incompatible seed hash: expected {}, got {}",
-                seed_hash,
-                sketch_b.seed_hash(),
-            )));
+        let union = self.compute_union(sketch_a, sketch_b)?;
+        if !union.entries.is_empty() && identical_sets(sketch_a, sketch_b, &union) {
+            return Ok(JaccardSimilarity::exact(1.0));
         }
 
         let sketch_a = KeySketchView::new(sketch_a);
         let sketch_b = KeySketchView::new(sketch_b);
-        let mut union = UnionState::new(
-            union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()),
-            ResizeFactor::X8,
-            1.0,
-            self.seed,
-            NoopMergePolicy,
-        );
-        union.update(&sketch_a)?;
-        union.update(&sketch_b)?;
-        let union = union.to_compact_parts(false);
-
-        if !union.entries.is_empty()
-            && union.entries.len() == sketch_a.num_retained()
-            && union.entries.len() == sketch_b.num_retained()
-            && union.theta == sketch_a.theta64()
-            && union.theta == sketch_b.theta64()
-        {
-            return Ok(JaccardSimilarity::exact(1.0));
-        }
-
         let union = CompactKeySketchView {
             entries: union.entries,
             theta: union.theta,
@@ -299,6 +269,75 @@ impl JaccardSimilarityOperator {
             union.theta64(),
         )
     }
+
+    pub(crate) fn exactly_equal<A, B>(&self, sketch_a: &A, sketch_b: &B) -> Result<bool, Error>
+    where
+        A: ThetaKeySketchView,
+        B: ThetaKeySketchView,
+    {
+        if sketch_a.is_empty() && sketch_b.is_empty() {
+            return Ok(true);
+        }
+        if sketch_a.is_empty() || sketch_b.is_empty() {
+            return Ok(false);
+        }
+
+        let union = self.compute_union(sketch_a, sketch_b)?;
+        Ok(identical_sets(sketch_a, sketch_b, &union))
+    }
+
+    fn compute_union<A, B>(
+        &self,
+        sketch_a: &A,
+        sketch_b: &B,
+    ) -> Result<CompactSketchParts<KeyEntry>, Error>
+    where
+        A: ThetaKeySketchView,
+        B: ThetaKeySketchView,
+    {
+        self.validate_seed_hash(sketch_a)?;
+        self.validate_seed_hash(sketch_b)?;
+
+        let sketch_a = KeySketchView::new(sketch_a);
+        let sketch_b = KeySketchView::new(sketch_b);
+        let mut union = UnionState::new(
+            union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()),
+            ResizeFactor::X8,
+            1.0,
+            self.seed,
+            NoopMergePolicy,
+        );
+        union.update(&sketch_a)?;
+        union.update(&sketch_b)?;
+        Ok(union.to_compact_parts(false))
+    }
+
+    fn validate_seed_hash<S: ThetaKeySketchView>(&self, sketch: &S) -> Result<(), Error> {
+        let expected = compute_seed_hash(self.seed);
+        if expected != sketch.seed_hash() {
+            return Err(Error::invalid_argument(format!(
+                "incompatible seed hash: expected {}, got {}",
+                expected,
+                sketch.seed_hash(),
+            )));
+        }
+        Ok(())
+    }
+}
+
+/// Returns whether both sketches have the same retained keys and theta.
+///
+/// When the union retains no additional keys and preserves both input theta values, each input
+/// contains exactly the same retained key set represented by the union.
+fn identical_sets<A, B>(sketch_a: &A, sketch_b: &B, union: &CompactSketchParts<KeyEntry>) -> bool
+where
+    A: ThetaKeySketchView,
+    B: ThetaKeySketchView,
+{
+    union.entries.len() == sketch_a.num_retained()
+        && union.entries.len() == sketch_b.num_retained()
+        && union.theta == sketch_a.theta64()
+        && union.theta == sketch_b.theta64()
 }
 
 fn sampling_adjuster(sampling_probability: f64) -> f64 {

datasketches/src/thetafamily/theta/jaccard_similarity.rs

@@ -74,4 +74,22 @@ impl ThetaJaccardSimilarity {
     ) -> Result<JaccardSimilarity, Error> {
         self.op.compute(sketch_a, sketch_b)
     }
+
+    /// Returns whether the two sketches are exactly equal.
+    ///
+    /// Two logically empty sketches compare equal, while exactly one logically empty sketch
+    /// compares unequal. Otherwise, the retained hashes and theta must match. This compares sketch
+    /// state, not the original input populations.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if both sketches are non-empty and either was built with a seed different
+    /// from this operator's configured seed.
+    pub fn exactly_equal<A: ThetaSketchView, B: ThetaSketchView>(
+        &self,
+        sketch_a: &A,
+        sketch_b: &B,
+    ) -> Result<bool, Error> {
+        self.op.exactly_equal(sketch_a, sketch_b)
+    }
 }

datasketches/src/thetafamily/tuple/jaccard_similarity.rs

@@ -79,4 +79,23 @@ impl TupleJaccardSimilarity {
     {
         self.op.compute(sketch_a, sketch_b)
     }
+
+    /// Returns whether the two sketches are exactly equal.
+    ///
+    /// Two logically empty sketches compare equal, while exactly one logically empty sketch
+    /// compares unequal. Otherwise, the retained hash keys and theta must match. Summary values do
+    /// not participate in the comparison. This compares sketch state, not the original input
+    /// populations.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if both sketches are non-empty and either was built with a seed different
+    /// from this operator's configured seed.
+    pub fn exactly_equal<A, B>(&self, sketch_a: &A, sketch_b: &B) -> Result<bool, Error>
+    where
+        A: TupleKeySketchView,
+        B: TupleKeySketchView,
+    {
+        self.op.exactly_equal(sketch_a, sketch_b)
+    }
 }

datasketches/tests/theta_test/jaccard_similarity.rs

@@ -60,11 +60,28 @@ fn test_empty() {
     let sketch_a = ThetaSketchBuilder::default().build();
     let sketch_b = ThetaSketchBuilder::default().build();
 
-    let jaccard = ThetaJaccardSimilarity::default()
-        .compute(&sketch_a, &sketch_b)
-        .unwrap();
+    let operator = ThetaJaccardSimilarity::default();
+    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
 
     assert_jaccard_exact(jaccard, 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+}
+
+#[test]
+fn test_exactly_equal() {
+    let empty = ThetaSketchBuilder::default().build();
+    let sketch_a = sketch_with_range(0, 1000);
+    let sketch_b = sketch_with_range(0, 1000);
+    let sketch_c = sketch_with_range(1000, 1000);
+    let compact_a = sketch_a.compact(true);
+    let compact_b = sketch_b.compact(true);
+
+    let operator = ThetaJaccardSimilarity::default();
+    assert!(!operator.exactly_equal(&empty, &sketch_a).unwrap());
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+    assert!(operator.exactly_equal(&sketch_a, &compact_b).unwrap());
+    assert!(operator.exactly_equal(&compact_a, &sketch_b).unwrap());
+    assert!(!operator.exactly_equal(&sketch_a, &sketch_c).unwrap());
 }
 
 #[test]
@@ -144,6 +161,7 @@ fn test_half_overlap_estimation_mode_custom_seed() {
 
 #[test]
 fn test_seed_mismatch() {
+    let empty = ThetaSketchBuilder::default().build();
     let mut sketch_a = ThetaSketchBuilder::default().build();
     sketch_a.update(1u64);
     let mut sketch_b = ThetaSketchBuilder::default().seed(123).build();
@@ -154,6 +172,16 @@ fn test_seed_mismatch() {
             .compute(&sketch_a, &sketch_b)
             .is_err()
     );
+    assert!(
+        ThetaJaccardSimilarity::default()
+            .exactly_equal(&sketch_a, &sketch_b)
+            .is_err()
+    );
+    assert!(
+        !ThetaJaccardSimilarity::default()
+            .exactly_equal(&empty, &sketch_b)
+            .unwrap()
+    );
 }
 
 #[test]
@@ -164,18 +192,25 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
     let mut sketch_b = ThetaSketchBuilder::default()
         .sampling_probability(1e-12)
         .build();
+    let mut different_theta = ThetaSketchBuilder::default()
+        .sampling_probability(2e-12)
+        .build();
     sketch_a.update("apple");
     sketch_b.update("banana");
+    different_theta.update("orange");
 
     assert!(!sketch_a.is_empty());
     assert!(!sketch_b.is_empty());
     assert_eq!(sketch_a.num_retained(), 0);
     assert_eq!(sketch_b.num_retained(), 0);
+    assert_eq!(different_theta.num_retained(), 0);
 
-    let jaccard = ThetaJaccardSimilarity::default()
-        .compute(&sketch_a, &sketch_b)
-        .unwrap();
+    let operator = ThetaJaccardSimilarity::default();
+    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
     assert_eq!(jaccard.lower_bound(), 0.0);
     assert_eq!(jaccard.estimate(), 0.5);
     assert_eq!(jaccard.upper_bound(), 1.0);
+
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
+    assert!(!operator.exactly_equal(&sketch_a, &different_theta).unwrap());
 }

datasketches/tests/tuple_test/jaccard_similarity.rs

@@ -47,11 +47,11 @@ fn test_empty() {
     let sketch_a = default_tuple_sketch_builder().build();
     let sketch_b = default_tuple_sketch_builder().build();
 
-    let jaccard = TupleJaccardSimilarity::default()
-        .compute(&sketch_a, &sketch_b)
-        .unwrap();
+    let operator = TupleJaccardSimilarity::default();
+    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
 
     assert_jaccard_exact(jaccard, 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
 }
 
 #[test]
@@ -66,11 +66,14 @@ fn test_summary_values_and_types_do_not_affect_similarity() {
     let operator = TupleJaccardSimilarity::default();
     let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
     assert_jaccard_exact(jaccard, 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
 
-    let jaccard = operator
-        .compute(&sketch_a.compact(true), &sketch_b.compact(true))
-        .unwrap();
+    let compact_a = sketch_a.compact(true);
+    let compact_b = sketch_b.compact(true);
+    let jaccard = operator.compute(&compact_a, &compact_b).unwrap();
     assert_jaccard_exact(jaccard, 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &compact_b).unwrap());
+    assert!(operator.exactly_equal(&compact_a, &sketch_b).unwrap());
 }
 
 #[test]
@@ -81,6 +84,7 @@ fn test_half_overlap_estimation_mode() {
     let operator = TupleJaccardSimilarity::default();
     let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
     assert_jaccard_estimate(jaccard, 0.33);
+    assert!(!operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
 
     let jaccard = operator
         .compute(&sketch_a.compact(true), &sketch_b.compact(true))
@@ -91,6 +95,7 @@ fn test_half_overlap_estimation_mode() {
 #[test]
 fn test_custom_seed_and_seed_mismatch() {
     let seed = 123;
+    let empty = default_tuple_sketch_builder().build();
     let mut sketch_a = TupleSketchBuilder::new(DefaultUpdatePolicy::<u64>::default())
         .seed(seed)
         .build();
@@ -102,15 +107,25 @@ fn test_custom_seed_and_seed_mismatch() {
         sketch_b.update(value, 2u64);
     }
 
-    let jaccard = TupleJaccardSimilarity::with_seed(seed)
-        .compute(&sketch_a, &sketch_b)
-        .unwrap();
+    let operator = TupleJaccardSimilarity::with_seed(seed);
+    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
     assert_jaccard_exact(jaccard, 1.0);
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
     assert!(
         TupleJaccardSimilarity::default()
             .compute(&sketch_a, &sketch_b)
             .is_err()
     );
+    assert!(
+        TupleJaccardSimilarity::default()
+            .exactly_equal(&sketch_a, &sketch_b)
+            .is_err()
+    );
+    assert!(
+        !TupleJaccardSimilarity::default()
+            .exactly_equal(&empty, &sketch_a)
+            .unwrap()
+    );
 }
 
 #[test]
@@ -129,10 +144,11 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() {
     assert_eq!(sketch_a.num_retained(), 0);
     assert_eq!(sketch_b.num_retained(), 0);
 
-    let jaccard = TupleJaccardSimilarity::default()
-        .compute(&sketch_a, &sketch_b)
-        .unwrap();
+    let operator = TupleJaccardSimilarity::default();
+    let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap();
     assert_eq!(jaccard.lower_bound(), 0.0);
     assert_eq!(jaccard.estimate(), 0.5);
     assert_eq!(jaccard.upper_bound(), 1.0);
+
+    assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap());
 }