diff --git a/datasketches/src/thetafamily/common/jaccard_similarity.rs b/datasketches/src/thetafamily/common/jaccard_similarity.rs index 4e97b30..8ad7bc0 100644 --- a/datasketches/src/thetafamily/common/jaccard_similarity.rs +++ b/datasketches/src/thetafamily/common/jaccard_similarity.rs @@ -25,6 +25,7 @@ use crate::thetacommon::binomial_bounds; use crate::thetacommon::constants::MAX_LG_K; use crate::thetacommon::constants::MAX_THETA; use crate::thetacommon::constants::MIN_LG_K; +use crate::thetacommon::hash_table::CompactSketchParts; use crate::thetacommon::intersection::IntersectionMergePolicy; use crate::thetacommon::intersection::IntersectionState; use crate::thetacommon::union::UnionMergePolicy; @@ -242,44 +243,13 @@ impl JaccardSimilarityOperator { return Ok(JaccardSimilarity::exact(0.0)); } - let seed_hash = compute_seed_hash(self.seed); - if seed_hash != sketch_a.seed_hash() { - return Err(Error::invalid_argument(format!( - "incompatible seed hash: expected {}, got {}", - seed_hash, - sketch_a.seed_hash(), - ))); - } - if seed_hash != sketch_b.seed_hash() { - return Err(Error::invalid_argument(format!( - "incompatible seed hash: expected {}, got {}", - seed_hash, - sketch_b.seed_hash(), - ))); + let union = self.compute_union(sketch_a, sketch_b)?; + if !union.entries.is_empty() && identical_sets(sketch_a, sketch_b, &union) { + return Ok(JaccardSimilarity::exact(1.0)); } let sketch_a = KeySketchView::new(sketch_a); let sketch_b = KeySketchView::new(sketch_b); - let mut union = UnionState::new( - union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()), - ResizeFactor::X8, - 1.0, - self.seed, - NoopMergePolicy, - ); - union.update(&sketch_a)?; - union.update(&sketch_b)?; - let union = union.to_compact_parts(false); - - if !union.entries.is_empty() - && union.entries.len() == sketch_a.num_retained() - && union.entries.len() == sketch_b.num_retained() - && union.theta == sketch_a.theta64() - && union.theta == sketch_b.theta64() - { - return Ok(JaccardSimilarity::exact(1.0)); - } - let union = CompactKeySketchView { entries: union.entries, theta: union.theta, @@ -299,6 +269,75 @@ impl JaccardSimilarityOperator { union.theta64(), ) } + + pub(crate) fn exactly_equal(&self, sketch_a: &A, sketch_b: &B) -> Result + where + A: ThetaKeySketchView, + B: ThetaKeySketchView, + { + if sketch_a.is_empty() && sketch_b.is_empty() { + return Ok(true); + } + if sketch_a.is_empty() || sketch_b.is_empty() { + return Ok(false); + } + + let union = self.compute_union(sketch_a, sketch_b)?; + Ok(identical_sets(sketch_a, sketch_b, &union)) + } + + fn compute_union( + &self, + sketch_a: &A, + sketch_b: &B, + ) -> Result, Error> + where + A: ThetaKeySketchView, + B: ThetaKeySketchView, + { + self.validate_seed_hash(sketch_a)?; + self.validate_seed_hash(sketch_b)?; + + let sketch_a = KeySketchView::new(sketch_a); + let sketch_b = KeySketchView::new(sketch_b); + let mut union = UnionState::new( + union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()), + ResizeFactor::X8, + 1.0, + self.seed, + NoopMergePolicy, + ); + union.update(&sketch_a)?; + union.update(&sketch_b)?; + Ok(union.to_compact_parts(false)) + } + + fn validate_seed_hash(&self, sketch: &S) -> Result<(), Error> { + let expected = compute_seed_hash(self.seed); + if expected != sketch.seed_hash() { + return Err(Error::invalid_argument(format!( + "incompatible seed hash: expected {}, got {}", + expected, + sketch.seed_hash(), + ))); + } + Ok(()) + } +} + +/// Returns whether both sketches have the same retained keys and theta. +/// +/// When the union retains no additional keys and preserves both input theta values, each input +/// contains exactly the same retained key set represented by the union. +fn identical_sets(sketch_a: &A, sketch_b: &B, union: &CompactSketchParts) -> bool +where + A: ThetaKeySketchView, + B: ThetaKeySketchView, +{ + union.entries.len() == sketch_a.num_retained() + && union.entries.len() == sketch_b.num_retained() + && union.theta == sketch_a.theta64() + && union.theta == sketch_b.theta64() } fn sampling_adjuster(sampling_probability: f64) -> f64 { diff --git a/datasketches/src/thetafamily/theta/jaccard_similarity.rs b/datasketches/src/thetafamily/theta/jaccard_similarity.rs index 83beff0..b32a922 100644 --- a/datasketches/src/thetafamily/theta/jaccard_similarity.rs +++ b/datasketches/src/thetafamily/theta/jaccard_similarity.rs @@ -74,4 +74,22 @@ impl ThetaJaccardSimilarity { ) -> Result { self.op.compute(sketch_a, sketch_b) } + + /// Returns whether the two sketches are exactly equal. + /// + /// Two logically empty sketches compare equal, while exactly one logically empty sketch + /// compares unequal. Otherwise, the retained hashes and theta must match. This compares sketch + /// state, not the original input populations. + /// + /// # Errors + /// + /// Returns an error if both sketches are non-empty and either was built with a seed different + /// from this operator's configured seed. + pub fn exactly_equal( + &self, + sketch_a: &A, + sketch_b: &B, + ) -> Result { + self.op.exactly_equal(sketch_a, sketch_b) + } } diff --git a/datasketches/src/thetafamily/tuple/jaccard_similarity.rs b/datasketches/src/thetafamily/tuple/jaccard_similarity.rs index 6ad3835..1f47b3c 100644 --- a/datasketches/src/thetafamily/tuple/jaccard_similarity.rs +++ b/datasketches/src/thetafamily/tuple/jaccard_similarity.rs @@ -79,4 +79,23 @@ impl TupleJaccardSimilarity { { self.op.compute(sketch_a, sketch_b) } + + /// Returns whether the two sketches are exactly equal. + /// + /// Two logically empty sketches compare equal, while exactly one logically empty sketch + /// compares unequal. Otherwise, the retained hash keys and theta must match. Summary values do + /// not participate in the comparison. This compares sketch state, not the original input + /// populations. + /// + /// # Errors + /// + /// Returns an error if both sketches are non-empty and either was built with a seed different + /// from this operator's configured seed. + pub fn exactly_equal(&self, sketch_a: &A, sketch_b: &B) -> Result + where + A: TupleKeySketchView, + B: TupleKeySketchView, + { + self.op.exactly_equal(sketch_a, sketch_b) + } } diff --git a/datasketches/tests/theta_test/jaccard_similarity.rs b/datasketches/tests/theta_test/jaccard_similarity.rs index 428989f..acdee30 100644 --- a/datasketches/tests/theta_test/jaccard_similarity.rs +++ b/datasketches/tests/theta_test/jaccard_similarity.rs @@ -60,11 +60,28 @@ fn test_empty() { let sketch_a = ThetaSketchBuilder::default().build(); let sketch_b = ThetaSketchBuilder::default().build(); - let jaccard = ThetaJaccardSimilarity::default() - .compute(&sketch_a, &sketch_b) - .unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); +} + +#[test] +fn test_exactly_equal() { + let empty = ThetaSketchBuilder::default().build(); + let sketch_a = sketch_with_range(0, 1000); + let sketch_b = sketch_with_range(0, 1000); + let sketch_c = sketch_with_range(1000, 1000); + let compact_a = sketch_a.compact(true); + let compact_b = sketch_b.compact(true); + + let operator = ThetaJaccardSimilarity::default(); + assert!(!operator.exactly_equal(&empty, &sketch_a).unwrap()); + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); + assert!(operator.exactly_equal(&sketch_a, &compact_b).unwrap()); + assert!(operator.exactly_equal(&compact_a, &sketch_b).unwrap()); + assert!(!operator.exactly_equal(&sketch_a, &sketch_c).unwrap()); } #[test] @@ -144,6 +161,7 @@ fn test_half_overlap_estimation_mode_custom_seed() { #[test] fn test_seed_mismatch() { + let empty = ThetaSketchBuilder::default().build(); let mut sketch_a = ThetaSketchBuilder::default().build(); sketch_a.update(1u64); let mut sketch_b = ThetaSketchBuilder::default().seed(123).build(); @@ -154,6 +172,16 @@ fn test_seed_mismatch() { .compute(&sketch_a, &sketch_b) .is_err() ); + assert!( + ThetaJaccardSimilarity::default() + .exactly_equal(&sketch_a, &sketch_b) + .is_err() + ); + assert!( + !ThetaJaccardSimilarity::default() + .exactly_equal(&empty, &sketch_b) + .unwrap() + ); } #[test] @@ -164,18 +192,25 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() { let mut sketch_b = ThetaSketchBuilder::default() .sampling_probability(1e-12) .build(); + let mut different_theta = ThetaSketchBuilder::default() + .sampling_probability(2e-12) + .build(); sketch_a.update("apple"); sketch_b.update("banana"); + different_theta.update("orange"); assert!(!sketch_a.is_empty()); assert!(!sketch_b.is_empty()); assert_eq!(sketch_a.num_retained(), 0); assert_eq!(sketch_b.num_retained(), 0); + assert_eq!(different_theta.num_retained(), 0); - let jaccard = ThetaJaccardSimilarity::default() - .compute(&sketch_a, &sketch_b) - .unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_eq!(jaccard.lower_bound(), 0.0); assert_eq!(jaccard.estimate(), 0.5); assert_eq!(jaccard.upper_bound(), 1.0); + + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); + assert!(!operator.exactly_equal(&sketch_a, &different_theta).unwrap()); } diff --git a/datasketches/tests/tuple_test/jaccard_similarity.rs b/datasketches/tests/tuple_test/jaccard_similarity.rs index d568285..af9342a 100644 --- a/datasketches/tests/tuple_test/jaccard_similarity.rs +++ b/datasketches/tests/tuple_test/jaccard_similarity.rs @@ -47,11 +47,11 @@ fn test_empty() { let sketch_a = default_tuple_sketch_builder().build(); let sketch_b = default_tuple_sketch_builder().build(); - let jaccard = TupleJaccardSimilarity::default() - .compute(&sketch_a, &sketch_b) - .unwrap(); + let operator = TupleJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); } #[test] @@ -66,11 +66,14 @@ fn test_summary_values_and_types_do_not_affect_similarity() { let operator = TupleJaccardSimilarity::default(); let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); - let jaccard = operator - .compute(&sketch_a.compact(true), &sketch_b.compact(true)) - .unwrap(); + let compact_a = sketch_a.compact(true); + let compact_b = sketch_b.compact(true); + let jaccard = operator.compute(&compact_a, &compact_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); + assert!(operator.exactly_equal(&sketch_a, &compact_b).unwrap()); + assert!(operator.exactly_equal(&compact_a, &sketch_b).unwrap()); } #[test] @@ -81,6 +84,7 @@ fn test_half_overlap_estimation_mode() { let operator = TupleJaccardSimilarity::default(); let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_estimate(jaccard, 0.33); + assert!(!operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); let jaccard = operator .compute(&sketch_a.compact(true), &sketch_b.compact(true)) @@ -91,6 +95,7 @@ fn test_half_overlap_estimation_mode() { #[test] fn test_custom_seed_and_seed_mismatch() { let seed = 123; + let empty = default_tuple_sketch_builder().build(); let mut sketch_a = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) .seed(seed) .build(); @@ -102,15 +107,25 @@ fn test_custom_seed_and_seed_mismatch() { sketch_b.update(value, 2u64); } - let jaccard = TupleJaccardSimilarity::with_seed(seed) - .compute(&sketch_a, &sketch_b) - .unwrap(); + let operator = TupleJaccardSimilarity::with_seed(seed); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); assert!( TupleJaccardSimilarity::default() .compute(&sketch_a, &sketch_b) .is_err() ); + assert!( + TupleJaccardSimilarity::default() + .exactly_equal(&sketch_a, &sketch_b) + .is_err() + ); + assert!( + !TupleJaccardSimilarity::default() + .exactly_equal(&empty, &sketch_a) + .unwrap() + ); } #[test] @@ -129,10 +144,11 @@ fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() { assert_eq!(sketch_a.num_retained(), 0); assert_eq!(sketch_b.num_retained(), 0); - let jaccard = TupleJaccardSimilarity::default() - .compute(&sketch_a, &sketch_b) - .unwrap(); + let operator = TupleJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_eq!(jaccard.lower_bound(), 0.0); assert_eq!(jaccard.estimate(), 0.5); assert_eq!(jaccard.upper_bound(), 1.0); + + assert!(operator.exactly_equal(&sketch_a, &sketch_b).unwrap()); }