From 373d3d7af435379bb8903a8273c23a25b69ed1af Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Thu, 2 Jul 2026 15:07:42 +0800 Subject: [PATCH 01/10] feat(theta): add jaccard similarity --- datasketches/src/theta/jaccard_similarity.rs | 323 ++++++++++++++++++ datasketches/src/theta/mod.rs | 3 + .../tests/theta_jaccard_similarity_test.rs | 147 ++++++++ 3 files changed, 473 insertions(+) create mode 100644 datasketches/src/theta/jaccard_similarity.rs create mode 100644 datasketches/tests/theta_jaccard_similarity_test.rs diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs new file mode 100644 index 00000000..090ba28e --- /dev/null +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Jaccard similarity for Theta sketches. + +use std::collections::BTreeSet; + +use crate::error::Error; +use crate::hash::DEFAULT_UPDATE_SEED; +use crate::hash::compute_seed_hash; +use crate::theta::CompactThetaSketch; +use crate::theta::ThetaIntersection; +use crate::theta::ThetaSketchView; + +const NUM_STD_DEVS: f64 = 2.0; + +/// Jaccard similarity result for two Theta sketches. +/// +/// The entries are lower bound, estimate, and upper bound, matching the C++ +/// `theta_jaccard_similarity::jaccard` result order. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct JaccardSimilarity { + /// Approximate lower bound for the Jaccard index. + pub lower_bound: f64, + /// Estimate of the Jaccard index. + pub estimate: f64, + /// Approximate upper bound for the Jaccard index. + pub upper_bound: f64, +} + +impl JaccardSimilarity { + fn exact(value: f64) -> Self { + Self { + lower_bound: value, + estimate: value, + upper_bound: value, + } + } +} + +/// Computes Jaccard similarity between Theta sketches. +pub struct ThetaJaccardSimilarity; + +impl ThetaJaccardSimilarity { + /// Computes the Jaccard similarity index with the default update seed. + /// + /// The Jaccard index is `|A intersection B| / |A union B|`. The returned value contains + /// lower bound, estimate, and upper bound. + pub fn jaccard( + sketch_a: &A, + sketch_b: &B, + ) -> Result { + Self::jaccard_with_seed(sketch_a, sketch_b, DEFAULT_UPDATE_SEED) + } + + /// Computes the Jaccard similarity index with an explicit update seed. + /// + /// Returns an error if a non-empty sketch was built with a different seed. + pub fn jaccard_with_seed( + sketch_a: &A, + sketch_b: &B, + seed: u64, + ) -> Result { + if sketch_a.is_empty() && sketch_b.is_empty() { + return Ok(JaccardSimilarity::exact(1.0)); + } + if sketch_a.is_empty() || sketch_b.is_empty() { + return Ok(JaccardSimilarity::exact(0.0)); + } + + let union = compute_union(sketch_a, sketch_b, seed)?; + if identical_sets(sketch_a, sketch_b, &union) { + return Ok(JaccardSimilarity::exact(1.0)); + } + + let mut intersection = ThetaIntersection::new(seed); + intersection.update(sketch_a)?; + intersection.update(sketch_b)?; + intersection.update(&union)?; + let intersection = intersection.result_with_ordered(false); + + ratio_bounds(&union, &intersection) + } +} + +fn compute_union( + sketch_a: &A, + sketch_b: &B, + seed: u64, +) -> Result { + let expected_seed_hash = compute_seed_hash(seed); + if !sketch_a.is_empty() && sketch_a.seed_hash() != expected_seed_hash { + return Err(Error::invalid_argument(format!( + "incompatible seed hash for sketch A: expected {}, got {}", + expected_seed_hash, + sketch_a.seed_hash() + ))); + } + if !sketch_b.is_empty() && sketch_b.seed_hash() != expected_seed_hash { + return Err(Error::invalid_argument(format!( + "incompatible seed hash for sketch B: expected {}, got {}", + expected_seed_hash, + sketch_b.seed_hash() + ))); + } + + let theta = sketch_a.theta64().min(sketch_b.theta64()); + let mut entries = BTreeSet::new(); + entries.extend(sketch_a.iter().filter(|&hash| hash < theta)); + entries.extend(sketch_b.iter().filter(|&hash| hash < theta)); + + Ok(CompactThetaSketch::from_parts( + entries.into_iter().collect(), + theta, + expected_seed_hash, + false, + false, + )) +} + +fn identical_sets( + sketch_a: &A, + sketch_b: &B, + union: &CompactThetaSketch, +) -> bool { + union.num_retained() == sketch_a.num_retained() + && union.num_retained() == sketch_b.num_retained() + && union.theta64() == sketch_a.theta64() + && union.theta64() == sketch_b.theta64() +} + +fn ratio_bounds( + sketch_a: &CompactThetaSketch, + sketch_b: &CompactThetaSketch, +) -> Result { + let theta_a = sketch_a.theta64(); + let theta_b = sketch_b.theta64(); + if theta_b > theta_a { + return Err(Error::invalid_argument(format!( + "theta_a must be <= theta_b: theta_a={theta_a}, theta_b={theta_b}" + ))); + } + + let count_b = sketch_b.num_retained() as u64; + let count_a = if theta_a == theta_b { + sketch_a.num_retained() as u64 + } else { + sketch_a.iter().filter(|&hash| hash < theta_b).count() as u64 + }; + + if count_a == 0 { + return Ok(JaccardSimilarity { + lower_bound: 0.0, + estimate: 0.5, + upper_bound: 1.0, + }); + } + + let f = sketch_b.theta(); + Ok(JaccardSimilarity { + lower_bound: lower_bound_for_b_over_a(count_a, count_b, f)?, + estimate: count_b as f64 / count_a as f64, + upper_bound: upper_bound_for_b_over_a(count_a, count_b, f)?, + }) +} + +fn lower_bound_for_b_over_a(a: u64, b: u64, f: f64) -> Result { + check_ratio_inputs(a, b, f)?; + if a == 0 { + return Ok(0.0); + } + if f == 1.0 { + return Ok(b as f64 / a as f64); + } + Ok(approximate_lower_bound_on_p( + a, + b, + NUM_STD_DEVS * hacky_adjuster(f), + )) +} + +fn upper_bound_for_b_over_a(a: u64, b: u64, f: f64) -> Result { + check_ratio_inputs(a, b, f)?; + if a == 0 { + return Ok(1.0); + } + if f == 1.0 { + return Ok(b as f64 / a as f64); + } + Ok(approximate_upper_bound_on_p( + a, + b, + NUM_STD_DEVS * hacky_adjuster(f), + )) +} + +fn check_ratio_inputs(a: u64, b: u64, f: f64) -> Result<(), Error> { + if a < b { + return Err(Error::invalid_argument(format!( + "a must be >= b: a = {a}, b = {b}" + ))); + } + if !(0.0..=1.0).contains(&f) || f == 0.0 { + return Err(Error::invalid_argument(format!( + "f must be in the range (0.0, 1.0], got {f}" + ))); + } + Ok(()) +} + +fn hacky_adjuster(f: f64) -> f64 { + let tmp = (1.0 - f).sqrt(); + if f <= 0.5 { + tmp + } else { + tmp + (0.01 * (f - 0.5)) + } +} + +fn approximate_lower_bound_on_p(n: u64, k: u64, num_std_devs: f64) -> f64 { + if n == 0 || k == 0 { + 0.0 + } else if k == 1 { + exact_lower_bound_on_p_k_eq_1(n, delta_of_num_stdevs(num_std_devs)) + } else if k == n { + exact_lower_bound_on_p_k_eq_n(n, delta_of_num_stdevs(num_std_devs)) + } else { + let x = abramowitz_stegun_formula_26p5p22((n - k) as f64 + 1.0, k as f64, -num_std_devs); + 1.0 - x + } +} + +fn approximate_upper_bound_on_p(n: u64, k: u64, num_std_devs: f64) -> f64 { + if n == 0 || k == n { + 1.0 + } else if k == n - 1 { + exact_upper_bound_on_p_k_eq_minusone(n, delta_of_num_stdevs(num_std_devs)) + } else if k == 0 { + exact_upper_bound_on_p_k_eq_zero(n, delta_of_num_stdevs(num_std_devs)) + } else { + let x = abramowitz_stegun_formula_26p5p22((n - k) as f64, k as f64 + 1.0, num_std_devs); + 1.0 - x + } +} + +fn delta_of_num_stdevs(kappa: f64) -> f64 { + normal_cdf(-kappa) +} + +fn normal_cdf(x: f64) -> f64 { + 0.5 * (1.0 + erf(x / 2.0_f64.sqrt())) +} + +fn erf(x: f64) -> f64 { + if x < 0.0 { + -erf_of_nonneg(-x) + } else { + erf_of_nonneg(x) + } +} + +fn erf_of_nonneg(x: f64) -> f64 { + let a1 = 0.0705230784; + let a2 = 0.0422820123; + let a3 = 0.0092705272; + let a4 = 0.0001520143; + let a5 = 0.0002765672; + let a6 = 0.0000430638; + let x2 = x * x; + let x3 = x2 * x; + let x4 = x2 * x2; + let x5 = x2 * x3; + let x6 = x3 * x3; + let sum = 1.0 + (a1 * x) + (a2 * x2) + (a3 * x3) + (a4 * x4) + (a5 * x5) + (a6 * x6); + let sum2 = sum * sum; + let sum4 = sum2 * sum2; + let sum8 = sum4 * sum4; + let sum16 = sum8 * sum8; + 1.0 - (1.0 / sum16) +} + +fn abramowitz_stegun_formula_26p5p22(a: f64, b: f64, yp: f64) -> f64 { + let b2m1 = (2.0 * b) - 1.0; + let a2m1 = (2.0 * a) - 1.0; + let lambda = ((yp * yp) - 3.0) / 6.0; + let htmp = (1.0 / a2m1) + (1.0 / b2m1); + let h = 2.0 / htmp; + let term1 = (yp * (h + lambda).sqrt()) / h; + let term2 = (1.0 / b2m1) - (1.0 / a2m1); + let term3 = (lambda + (5.0 / 6.0)) - (2.0 / (3.0 * h)); + let w = term1 - (term2 * term3); + a / (a + (b * (2.0 * w).exp())) +} + +fn exact_upper_bound_on_p_k_eq_zero(n: u64, delta: f64) -> f64 { + 1.0 - delta.powf(1.0 / n as f64) +} + +fn exact_lower_bound_on_p_k_eq_n(n: u64, delta: f64) -> f64 { + delta.powf(1.0 / n as f64) +} + +fn exact_lower_bound_on_p_k_eq_1(n: u64, delta: f64) -> f64 { + 1.0 - (1.0 - delta).powf(1.0 / n as f64) +} + +fn exact_upper_bound_on_p_k_eq_minusone(n: u64, delta: f64) -> f64 { + (1.0 - delta).powf(1.0 / n as f64) +} diff --git a/datasketches/src/theta/mod.rs b/datasketches/src/theta/mod.rs index 03b5e2a6..dc6523ba 100644 --- a/datasketches/src/theta/mod.rs +++ b/datasketches/src/theta/mod.rs @@ -42,10 +42,13 @@ mod bit_pack; mod hash_table; mod intersection; +mod jaccard_similarity; mod serialization; mod sketch; pub use self::intersection::ThetaIntersection; +pub use self::jaccard_similarity::JaccardSimilarity; +pub use self::jaccard_similarity::ThetaJaccardSimilarity; pub use self::sketch::CompactThetaSketch; pub use self::sketch::ThetaSketch; pub use self::sketch::ThetaSketchBuilder; diff --git a/datasketches/tests/theta_jaccard_similarity_test.rs b/datasketches/tests/theta_jaccard_similarity_test.rs new file mode 100644 index 00000000..aa808d16 --- /dev/null +++ b/datasketches/tests/theta_jaccard_similarity_test.rs @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![cfg(feature = "theta")] + +use datasketches::theta::ThetaJaccardSimilarity; +use datasketches::theta::ThetaSketch; + +fn assert_jaccard_exact(actual: datasketches::theta::JaccardSimilarity, expected: f64) { + assert_eq!(actual.lower_bound, expected); + assert_eq!(actual.estimate, expected); + assert_eq!(actual.upper_bound, expected); +} + +fn assert_close(actual: f64, expected: f64, margin: f64) { + assert!( + (actual - expected).abs() <= margin, + "actual={actual}, expected={expected}, margin={margin}" + ); +} + +fn sketch_with_range(start: u64, count: u64) -> ThetaSketch { + let mut sketch = ThetaSketch::builder().build(); + for value in start..start + count { + sketch.update(value); + } + sketch +} + +fn sketch_with_range_and_seed(start: u64, count: u64, seed: u64) -> ThetaSketch { + let mut sketch = ThetaSketch::builder().seed(seed).build(); + for value in start..start + count { + sketch.update(value); + } + sketch +} + +#[test] +fn test_empty() { + let sketch_a = ThetaSketch::builder().build(); + let sketch_b = ThetaSketch::builder().build(); + + let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + + assert_jaccard_exact(jaccard, 1.0); +} + +#[test] +fn test_same_sketch_exact_mode() { + let sketch = sketch_with_range(0, 1000); + + let jaccard = ThetaJaccardSimilarity::jaccard(&sketch, &sketch).unwrap(); + assert_jaccard_exact(jaccard, 1.0); + + let jaccard = + ThetaJaccardSimilarity::jaccard(&sketch.compact(true), &sketch.compact(true)).unwrap(); + assert_jaccard_exact(jaccard, 1.0); +} + +#[test] +fn test_full_overlap_exact_mode() { + let sketch_a = sketch_with_range(0, 1000); + let sketch_b = sketch_with_range(0, 1000); + + let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + assert_jaccard_exact(jaccard, 1.0); + + let jaccard = + ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + assert_jaccard_exact(jaccard, 1.0); +} + +#[test] +fn test_disjoint_exact_mode() { + let sketch_a = sketch_with_range(0, 1000); + let sketch_b = sketch_with_range(1000, 1000); + + let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + assert_jaccard_exact(jaccard, 0.0); + + let jaccard = + ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + assert_jaccard_exact(jaccard, 0.0); +} + +#[test] +fn test_half_overlap_estimation_mode() { + let sketch_a = sketch_with_range(0, 10000); + let sketch_b = sketch_with_range(5000, 10000); + + let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + assert_close(jaccard.lower_bound, 0.33, 0.01); + assert_close(jaccard.estimate, 0.33, 0.01); + assert_close(jaccard.upper_bound, 0.33, 0.01); + + let jaccard = + ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + assert_close(jaccard.lower_bound, 0.33, 0.01); + assert_close(jaccard.estimate, 0.33, 0.01); + assert_close(jaccard.upper_bound, 0.33, 0.01); +} + +#[test] +fn test_half_overlap_estimation_mode_custom_seed() { + let seed = 123; + let sketch_a = sketch_with_range_and_seed(0, 10000, seed); + let sketch_b = sketch_with_range_and_seed(5000, 10000, seed); + + let jaccard = ThetaJaccardSimilarity::jaccard_with_seed(&sketch_a, &sketch_b, seed).unwrap(); + assert_close(jaccard.lower_bound, 0.33, 0.01); + assert_close(jaccard.estimate, 0.33, 0.01); + assert_close(jaccard.upper_bound, 0.33, 0.01); + + let jaccard = ThetaJaccardSimilarity::jaccard_with_seed( + &sketch_a.compact(true), + &sketch_b.compact(true), + seed, + ) + .unwrap(); + assert_close(jaccard.lower_bound, 0.33, 0.01); + assert_close(jaccard.estimate, 0.33, 0.01); + assert_close(jaccard.upper_bound, 0.33, 0.01); +} + +#[test] +fn test_seed_mismatch() { + let mut sketch_a = ThetaSketch::builder().build(); + sketch_a.update(1u64); + let mut sketch_b = ThetaSketch::builder().seed(123).build(); + sketch_b.update(1u64); + + assert!(ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).is_err()); +} From e53af3bd768a943ca76d4a5317d704d766056806 Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Thu, 2 Jul 2026 15:20:31 +0800 Subject: [PATCH 02/10] docs(theta): clarify jaccard similarity --- datasketches/src/theta/jaccard_similarity.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index 090ba28e..ddbb9121 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -16,6 +16,10 @@ // under the License. //! Jaccard similarity for Theta sketches. +//! +//! The Jaccard similarity index is `J(A, B) = |A intersection B| / |A union B|`. +//! It measures how similar two sketches are: `1.0` means they are considered equal, +//! `0.0` means they are disjoint, and `0.95` means the overlap is 95% of the union. use std::collections::BTreeSet; @@ -31,7 +35,8 @@ const NUM_STD_DEVS: f64 = 2.0; /// Jaccard similarity result for two Theta sketches. /// /// The entries are lower bound, estimate, and upper bound, matching the C++ -/// `theta_jaccard_similarity::jaccard` result order. +/// `theta_jaccard_similarity::jaccard` result order. The bounds use a 95.4% +/// confidence interval, equivalent to +/- 2 standard deviations. #[derive(Clone, Copy, Debug, PartialEq)] pub struct JaccardSimilarity { /// Approximate lower bound for the Jaccard index. @@ -58,8 +63,9 @@ pub struct ThetaJaccardSimilarity; impl ThetaJaccardSimilarity { /// Computes the Jaccard similarity index with the default update seed. /// - /// The Jaccard index is `|A intersection B| / |A union B|`. The returned value contains - /// lower bound, estimate, and upper bound. + /// The returned value contains lower bound, estimate, and upper bound. For very large + /// sketches, where the configured nominal entries are `2^25` or `2^26`, this method may + /// produce unstable results. pub fn jaccard( sketch_a: &A, sketch_b: &B, @@ -69,6 +75,10 @@ impl ThetaJaccardSimilarity { /// Computes the Jaccard similarity index with an explicit update seed. /// + /// The returned value contains lower bound, estimate, and upper bound. For very large + /// sketches, where the configured nominal entries are `2^25` or `2^26`, this method may + /// produce unstable results. + /// /// Returns an error if a non-empty sketch was built with a different seed. pub fn jaccard_with_seed( sketch_a: &A, @@ -90,6 +100,8 @@ impl ThetaJaccardSimilarity { let mut intersection = ThetaIntersection::new(seed); intersection.update(sketch_a)?; intersection.update(sketch_b)?; + // Match the C++ implementation: intersect with the union to ensure that the + // final intersection sketch is a subset of the denominator sketch. intersection.update(&union)?; let intersection = intersection.result_with_ordered(false); From 75c384f4d1e7bcd6f65f884a478cbd367fc4561b Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Thu, 2 Jul 2026 15:31:11 +0800 Subject: [PATCH 03/10] docs(theta): avoid cpp-specific jaccard wording --- datasketches/src/theta/jaccard_similarity.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index ddbb9121..3f78dab0 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -100,8 +100,8 @@ impl ThetaJaccardSimilarity { let mut intersection = ThetaIntersection::new(seed); intersection.update(sketch_a)?; intersection.update(sketch_b)?; - // Match the C++ implementation: intersect with the union to ensure that the - // final intersection sketch is a subset of the denominator sketch. + // Ensure the numerator sketch is a subset of the denominator sketch used by + // the ratio bounds calculation. intersection.update(&union)?; let intersection = intersection.result_with_ordered(false); From fd02014f7138cd32fcc6e4c52a88e8db40a7c767 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 2 Jul 2026 18:14:17 +0800 Subject: [PATCH 04/10] fixup Signed-off-by: tison --- typos.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typos.toml b/typos.toml index c9b22da2..8d4c3e4d 100644 --- a/typos.toml +++ b/typos.toml @@ -16,8 +16,8 @@ # under the License. [default.extend-words] -# False-Positive Abbreviations "PREINTS" = "PREINTS" +"htmp" = "htmp" [files] extend-exclude = [] From 915e51814b55b4d41e5ea85e3d494c823cf1b05b Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Sat, 4 Jul 2026 14:40:30 +0800 Subject: [PATCH 05/10] refactor(theta): extract jaccard union utility --- datasketches/src/theta/jaccard_similarity.rs | 41 +------ datasketches/src/theta/mod.rs | 1 + datasketches/src/theta/union.rs | 113 +++++++++++++++++++ 3 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 datasketches/src/theta/union.rs diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index 3f78dab0..e80c8a16 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -21,14 +21,12 @@ //! It measures how similar two sketches are: `1.0` means they are considered equal, //! `0.0` means they are disjoint, and `0.95` means the overlap is 95% of the union. -use std::collections::BTreeSet; - use crate::error::Error; use crate::hash::DEFAULT_UPDATE_SEED; -use crate::hash::compute_seed_hash; use crate::theta::CompactThetaSketch; use crate::theta::ThetaIntersection; use crate::theta::ThetaSketchView; +use crate::theta::union::ThetaUnion; const NUM_STD_DEVS: f64 = 2.0; @@ -92,7 +90,7 @@ impl ThetaJaccardSimilarity { return Ok(JaccardSimilarity::exact(0.0)); } - let union = compute_union(sketch_a, sketch_b, seed)?; + let union = ThetaUnion::compute(sketch_a, sketch_b, seed)?; if identical_sets(sketch_a, sketch_b, &union) { return Ok(JaccardSimilarity::exact(1.0)); } @@ -109,41 +107,6 @@ impl ThetaJaccardSimilarity { } } -fn compute_union( - sketch_a: &A, - sketch_b: &B, - seed: u64, -) -> Result { - let expected_seed_hash = compute_seed_hash(seed); - if !sketch_a.is_empty() && sketch_a.seed_hash() != expected_seed_hash { - return Err(Error::invalid_argument(format!( - "incompatible seed hash for sketch A: expected {}, got {}", - expected_seed_hash, - sketch_a.seed_hash() - ))); - } - if !sketch_b.is_empty() && sketch_b.seed_hash() != expected_seed_hash { - return Err(Error::invalid_argument(format!( - "incompatible seed hash for sketch B: expected {}, got {}", - expected_seed_hash, - sketch_b.seed_hash() - ))); - } - - let theta = sketch_a.theta64().min(sketch_b.theta64()); - let mut entries = BTreeSet::new(); - entries.extend(sketch_a.iter().filter(|&hash| hash < theta)); - entries.extend(sketch_b.iter().filter(|&hash| hash < theta)); - - Ok(CompactThetaSketch::from_parts( - entries.into_iter().collect(), - theta, - expected_seed_hash, - false, - false, - )) -} - fn identical_sets( sketch_a: &A, sketch_b: &B, diff --git a/datasketches/src/theta/mod.rs b/datasketches/src/theta/mod.rs index dc6523ba..4bd466e2 100644 --- a/datasketches/src/theta/mod.rs +++ b/datasketches/src/theta/mod.rs @@ -45,6 +45,7 @@ mod intersection; mod jaccard_similarity; mod serialization; mod sketch; +mod union; pub use self::intersection::ThetaIntersection; pub use self::jaccard_similarity::JaccardSimilarity; diff --git a/datasketches/src/theta/union.rs b/datasketches/src/theta/union.rs new file mode 100644 index 00000000..cbf5ad10 --- /dev/null +++ b/datasketches/src/theta/union.rs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::BTreeSet; + +use crate::error::Error; +use crate::hash::compute_seed_hash; +use crate::theta::CompactThetaSketch; +use crate::theta::ThetaSketchView; + +pub(super) struct ThetaUnion; + +impl ThetaUnion { + pub(super) fn compute( + sketch_a: &A, + sketch_b: &B, + seed: u64, + ) -> Result { + let seed_hash = compute_seed_hash(seed); + validate_seed_hash(sketch_a, seed_hash, "sketch A")?; + validate_seed_hash(sketch_b, seed_hash, "sketch B")?; + + let theta = sketch_a.theta64().min(sketch_b.theta64()); + let mut entries = BTreeSet::new(); + entries.extend(sketch_a.iter().filter(|&hash| hash < theta)); + entries.extend(sketch_b.iter().filter(|&hash| hash < theta)); + + Ok(CompactThetaSketch::from_parts( + entries.into_iter().collect(), + theta, + seed_hash, + false, + sketch_a.is_empty() && sketch_b.is_empty(), + )) + } +} + +fn validate_seed_hash( + sketch: &S, + expected_seed_hash: u16, + label: &str, +) -> Result<(), Error> { + if !sketch.is_empty() && sketch.seed_hash() != expected_seed_hash { + return Err(Error::invalid_argument(format!( + "incompatible seed hash for {label}: expected {}, got {}", + expected_seed_hash, + sketch.seed_hash() + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::hash::DEFAULT_UPDATE_SEED; + use crate::theta::ThetaSketch; + use crate::theta::union::ThetaUnion; + + fn sketch_with_range(start: u64, count: u64) -> ThetaSketch { + let mut sketch = ThetaSketch::builder().build(); + for value in start..start + count { + sketch.update(value); + } + sketch + } + + #[test] + fn exact_mode_half_overlap() { + let sketch_a = sketch_with_range(0, 1000); + let sketch_b = sketch_with_range(500, 1000); + + let union = ThetaUnion::compute(&sketch_a, &sketch_b, DEFAULT_UPDATE_SEED).unwrap(); + + assert!(!union.is_empty()); + assert!(!union.is_estimation_mode()); + assert_eq!(union.estimate(), 1500.0); + } + + #[test] + fn empty_inputs_produce_empty_union() { + let sketch_a = ThetaSketch::builder().build(); + let sketch_b = ThetaSketch::builder().build(); + + let union = ThetaUnion::compute(&sketch_a, &sketch_b, DEFAULT_UPDATE_SEED).unwrap(); + + assert!(union.is_empty()); + assert!(!union.is_estimation_mode()); + assert_eq!(union.num_retained(), 0); + } + + #[test] + fn seed_mismatch_on_non_empty_sketch_returns_error() { + let mut sketch_a = ThetaSketch::builder().seed(123).build(); + sketch_a.update(1u64); + let sketch_b = ThetaSketch::builder().build(); + + assert!(ThetaUnion::compute(&sketch_a, &sketch_b, DEFAULT_UPDATE_SEED).is_err()); + } +} From 57d27af4cd38c76636079260a86a502d15baa612 Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Sun, 5 Jul 2026 19:17:37 +0800 Subject: [PATCH 06/10] fix(theta): refine jaccard API --- .../src/common/bounds_binomial_proportions.rs | 161 ++++++++++++++++++ datasketches/src/common/mod.rs | 2 + datasketches/src/theta/jaccard_similarity.rs | 161 +++++------------- datasketches/src/theta/mod.rs | 1 - .../tests/theta_jaccard_similarity_test.rs | 57 +++---- 5 files changed, 230 insertions(+), 152 deletions(-) create mode 100644 datasketches/src/common/bounds_binomial_proportions.rs diff --git a/datasketches/src/common/bounds_binomial_proportions.rs b/datasketches/src/common/bounds_binomial_proportions.rs new file mode 100644 index 00000000..ba603a07 --- /dev/null +++ b/datasketches/src/common/bounds_binomial_proportions.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::error::Error; + +/// Computes an approximate lower bound for an unknown binomial proportion. +pub(crate) fn approximate_lower_bound_on_p( + n: u64, + k: u64, + num_std_devs: f64, +) -> Result { + check_inputs(n, k)?; + if n == 0 || k == 0 { + Ok(0.0) + } else if k == 1 { + Ok(exact_lower_bound_on_p_k_eq_1( + n, + delta_of_num_stdevs(num_std_devs), + )) + } else if k == n { + Ok(exact_lower_bound_on_p_k_eq_n( + n, + delta_of_num_stdevs(num_std_devs), + )) + } else { + let x = abramowitz_stegun_formula_26p5p22((n - k) as f64 + 1.0, k as f64, -num_std_devs); + Ok(1.0 - x) + } +} + +/// Computes an approximate upper bound for an unknown binomial proportion. +pub(crate) fn approximate_upper_bound_on_p( + n: u64, + k: u64, + num_std_devs: f64, +) -> Result { + check_inputs(n, k)?; + if n == 0 || k == n { + Ok(1.0) + } else if k == n - 1 { + Ok(exact_upper_bound_on_p_k_eq_minusone( + n, + delta_of_num_stdevs(num_std_devs), + )) + } else if k == 0 { + Ok(exact_upper_bound_on_p_k_eq_zero( + n, + delta_of_num_stdevs(num_std_devs), + )) + } else { + let x = abramowitz_stegun_formula_26p5p22((n - k) as f64, k as f64 + 1.0, num_std_devs); + Ok(1.0 - x) + } +} + +fn check_inputs(n: u64, k: u64) -> Result<(), Error> { + if k > n { + return Err(Error::invalid_argument(format!( + "k cannot exceed n: k={k}, n={n}" + ))); + } + Ok(()) +} + +fn delta_of_num_stdevs(kappa: f64) -> f64 { + normal_cdf(-kappa) +} + +fn normal_cdf(x: f64) -> f64 { + 0.5 * (1.0 + erf(x / 2.0_f64.sqrt())) +} + +fn erf(x: f64) -> f64 { + if x < 0.0 { + -erf_of_nonneg(-x) + } else { + erf_of_nonneg(x) + } +} + +fn erf_of_nonneg(x: f64) -> f64 { + let a1 = 0.0705230784; + let a2 = 0.0422820123; + let a3 = 0.0092705272; + let a4 = 0.0001520143; + let a5 = 0.0002765672; + let a6 = 0.0000430638; + let x2 = x * x; + let x3 = x2 * x; + let x4 = x2 * x2; + let x5 = x2 * x3; + let x6 = x3 * x3; + let sum = 1.0 + (a1 * x) + (a2 * x2) + (a3 * x3) + (a4 * x4) + (a5 * x5) + (a6 * x6); + let sum2 = sum * sum; + let sum4 = sum2 * sum2; + let sum8 = sum4 * sum4; + let sum16 = sum8 * sum8; + 1.0 - (1.0 / sum16) +} + +fn abramowitz_stegun_formula_26p5p22(a: f64, b: f64, yp: f64) -> f64 { + let b2m1 = (2.0 * b) - 1.0; + let a2m1 = (2.0 * a) - 1.0; + let lambda = ((yp * yp) - 3.0) / 6.0; + let htmp = (1.0 / a2m1) + (1.0 / b2m1); + let h = 2.0 / htmp; + let term1 = (yp * (h + lambda).sqrt()) / h; + let term2 = (1.0 / b2m1) - (1.0 / a2m1); + let term3 = (lambda + (5.0 / 6.0)) - (2.0 / (3.0 * h)); + let w = term1 - (term2 * term3); + a / (a + (b * (2.0 * w).exp())) +} + +fn exact_upper_bound_on_p_k_eq_zero(n: u64, delta: f64) -> f64 { + 1.0 - delta.powf(1.0 / n as f64) +} + +fn exact_lower_bound_on_p_k_eq_n(n: u64, delta: f64) -> f64 { + delta.powf(1.0 / n as f64) +} + +fn exact_lower_bound_on_p_k_eq_1(n: u64, delta: f64) -> f64 { + 1.0 - (1.0 - delta).powf(1.0 / n as f64) +} + +fn exact_upper_bound_on_p_k_eq_minusone(n: u64, delta: f64) -> f64 { + (1.0 - delta).powf(1.0 / n as f64) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_invalid_counts() { + assert!(approximate_lower_bound_on_p(1, 2, 2.0).is_err()); + assert!(approximate_upper_bound_on_p(1, 2, 2.0).is_err()); + } + + #[test] + fn computes_exact_edge_cases() { + assert_eq!(approximate_lower_bound_on_p(0, 0, 2.0).unwrap(), 0.0); + assert_eq!(approximate_upper_bound_on_p(0, 0, 2.0).unwrap(), 1.0); + assert_eq!(approximate_lower_bound_on_p(10, 0, 2.0).unwrap(), 0.0); + assert_eq!(approximate_upper_bound_on_p(10, 10, 2.0).unwrap(), 1.0); + } +} diff --git a/datasketches/src/common/mod.rs b/datasketches/src/common/mod.rs index 503df720..d8469416 100644 --- a/datasketches/src/common/mod.rs +++ b/datasketches/src/common/mod.rs @@ -24,5 +24,7 @@ pub use self::resize::ResizeFactor; #[cfg(feature = "theta")] pub(crate) mod binomial_bounds; +#[cfg(feature = "theta")] +pub(crate) mod bounds_binomial_proportions; #[cfg(feature = "cpc")] pub(crate) mod inv_pow2_table; diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index e80c8a16..05d16126 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -21,6 +21,7 @@ //! It measures how similar two sketches are: `1.0` means they are considered equal, //! `0.0` means they are disjoint, and `0.95` means the overlap is 95% of the union. +use crate::common::bounds_binomial_proportions; use crate::error::Error; use crate::hash::DEFAULT_UPDATE_SEED; use crate::theta::CompactThetaSketch; @@ -37,38 +38,22 @@ const NUM_STD_DEVS: f64 = 2.0; /// confidence interval, equivalent to +/- 2 standard deviations. #[derive(Clone, Copy, Debug, PartialEq)] pub struct JaccardSimilarity { - /// Approximate lower bound for the Jaccard index. - pub lower_bound: f64, - /// Estimate of the Jaccard index. - pub estimate: f64, - /// Approximate upper bound for the Jaccard index. - pub upper_bound: f64, + lower_bound: f64, + estimate: f64, + upper_bound: f64, } impl JaccardSimilarity { - fn exact(value: f64) -> Self { - Self { - lower_bound: value, - estimate: value, - upper_bound: value, - } - } -} - -/// Computes Jaccard similarity between Theta sketches. -pub struct ThetaJaccardSimilarity; - -impl ThetaJaccardSimilarity { /// Computes the Jaccard similarity index with the default update seed. /// /// The returned value contains lower bound, estimate, and upper bound. For very large /// sketches, where the configured nominal entries are `2^25` or `2^26`, this method may /// produce unstable results. - pub fn jaccard( + pub fn between( sketch_a: &A, sketch_b: &B, - ) -> Result { - Self::jaccard_with_seed(sketch_a, sketch_b, DEFAULT_UPDATE_SEED) + ) -> Result { + Self::between_with_seed(sketch_a, sketch_b, DEFAULT_UPDATE_SEED) } /// Computes the Jaccard similarity index with an explicit update seed. @@ -78,21 +63,21 @@ impl ThetaJaccardSimilarity { /// produce unstable results. /// /// Returns an error if a non-empty sketch was built with a different seed. - pub fn jaccard_with_seed( + pub fn between_with_seed( sketch_a: &A, sketch_b: &B, seed: u64, - ) -> Result { + ) -> Result { if sketch_a.is_empty() && sketch_b.is_empty() { - return Ok(JaccardSimilarity::exact(1.0)); + return Ok(Self::exact(1.0)); } if sketch_a.is_empty() || sketch_b.is_empty() { - return Ok(JaccardSimilarity::exact(0.0)); + return Ok(Self::exact(0.0)); } let union = ThetaUnion::compute(sketch_a, sketch_b, seed)?; if identical_sets(sketch_a, sketch_b, &union) { - return Ok(JaccardSimilarity::exact(1.0)); + return Ok(Self::exact(1.0)); } let mut intersection = ThetaIntersection::new(seed); @@ -105,6 +90,29 @@ impl ThetaJaccardSimilarity { ratio_bounds(&union, &intersection) } + + /// Returns the approximate lower bound for the Jaccard index. + pub fn lower_bound(&self) -> f64 { + self.lower_bound + } + + /// Returns the estimate of the Jaccard index. + pub fn estimate(&self) -> f64 { + self.estimate + } + + /// Returns the approximate upper bound for the Jaccard index. + pub fn upper_bound(&self) -> f64 { + self.upper_bound + } + + fn exact(value: f64) -> Self { + Self { + lower_bound: value, + estimate: value, + upper_bound: value, + } + } } fn identical_sets( @@ -161,11 +169,11 @@ fn lower_bound_for_b_over_a(a: u64, b: u64, f: f64) -> Result { if f == 1.0 { return Ok(b as f64 / a as f64); } - Ok(approximate_lower_bound_on_p( + bounds_binomial_proportions::approximate_lower_bound_on_p( a, b, NUM_STD_DEVS * hacky_adjuster(f), - )) + ) } fn upper_bound_for_b_over_a(a: u64, b: u64, f: f64) -> Result { @@ -176,11 +184,11 @@ fn upper_bound_for_b_over_a(a: u64, b: u64, f: f64) -> Result { if f == 1.0 { return Ok(b as f64 / a as f64); } - Ok(approximate_upper_bound_on_p( + bounds_binomial_proportions::approximate_upper_bound_on_p( a, b, NUM_STD_DEVS * hacky_adjuster(f), - )) + ) } fn check_ratio_inputs(a: u64, b: u64, f: f64) -> Result<(), Error> { @@ -205,94 +213,3 @@ fn hacky_adjuster(f: f64) -> f64 { tmp + (0.01 * (f - 0.5)) } } - -fn approximate_lower_bound_on_p(n: u64, k: u64, num_std_devs: f64) -> f64 { - if n == 0 || k == 0 { - 0.0 - } else if k == 1 { - exact_lower_bound_on_p_k_eq_1(n, delta_of_num_stdevs(num_std_devs)) - } else if k == n { - exact_lower_bound_on_p_k_eq_n(n, delta_of_num_stdevs(num_std_devs)) - } else { - let x = abramowitz_stegun_formula_26p5p22((n - k) as f64 + 1.0, k as f64, -num_std_devs); - 1.0 - x - } -} - -fn approximate_upper_bound_on_p(n: u64, k: u64, num_std_devs: f64) -> f64 { - if n == 0 || k == n { - 1.0 - } else if k == n - 1 { - exact_upper_bound_on_p_k_eq_minusone(n, delta_of_num_stdevs(num_std_devs)) - } else if k == 0 { - exact_upper_bound_on_p_k_eq_zero(n, delta_of_num_stdevs(num_std_devs)) - } else { - let x = abramowitz_stegun_formula_26p5p22((n - k) as f64, k as f64 + 1.0, num_std_devs); - 1.0 - x - } -} - -fn delta_of_num_stdevs(kappa: f64) -> f64 { - normal_cdf(-kappa) -} - -fn normal_cdf(x: f64) -> f64 { - 0.5 * (1.0 + erf(x / 2.0_f64.sqrt())) -} - -fn erf(x: f64) -> f64 { - if x < 0.0 { - -erf_of_nonneg(-x) - } else { - erf_of_nonneg(x) - } -} - -fn erf_of_nonneg(x: f64) -> f64 { - let a1 = 0.0705230784; - let a2 = 0.0422820123; - let a3 = 0.0092705272; - let a4 = 0.0001520143; - let a5 = 0.0002765672; - let a6 = 0.0000430638; - let x2 = x * x; - let x3 = x2 * x; - let x4 = x2 * x2; - let x5 = x2 * x3; - let x6 = x3 * x3; - let sum = 1.0 + (a1 * x) + (a2 * x2) + (a3 * x3) + (a4 * x4) + (a5 * x5) + (a6 * x6); - let sum2 = sum * sum; - let sum4 = sum2 * sum2; - let sum8 = sum4 * sum4; - let sum16 = sum8 * sum8; - 1.0 - (1.0 / sum16) -} - -fn abramowitz_stegun_formula_26p5p22(a: f64, b: f64, yp: f64) -> f64 { - let b2m1 = (2.0 * b) - 1.0; - let a2m1 = (2.0 * a) - 1.0; - let lambda = ((yp * yp) - 3.0) / 6.0; - let htmp = (1.0 / a2m1) + (1.0 / b2m1); - let h = 2.0 / htmp; - let term1 = (yp * (h + lambda).sqrt()) / h; - let term2 = (1.0 / b2m1) - (1.0 / a2m1); - let term3 = (lambda + (5.0 / 6.0)) - (2.0 / (3.0 * h)); - let w = term1 - (term2 * term3); - a / (a + (b * (2.0 * w).exp())) -} - -fn exact_upper_bound_on_p_k_eq_zero(n: u64, delta: f64) -> f64 { - 1.0 - delta.powf(1.0 / n as f64) -} - -fn exact_lower_bound_on_p_k_eq_n(n: u64, delta: f64) -> f64 { - delta.powf(1.0 / n as f64) -} - -fn exact_lower_bound_on_p_k_eq_1(n: u64, delta: f64) -> f64 { - 1.0 - (1.0 - delta).powf(1.0 / n as f64) -} - -fn exact_upper_bound_on_p_k_eq_minusone(n: u64, delta: f64) -> f64 { - (1.0 - delta).powf(1.0 / n as f64) -} diff --git a/datasketches/src/theta/mod.rs b/datasketches/src/theta/mod.rs index 4bd466e2..4ea38a98 100644 --- a/datasketches/src/theta/mod.rs +++ b/datasketches/src/theta/mod.rs @@ -49,7 +49,6 @@ mod union; pub use self::intersection::ThetaIntersection; pub use self::jaccard_similarity::JaccardSimilarity; -pub use self::jaccard_similarity::ThetaJaccardSimilarity; pub use self::sketch::CompactThetaSketch; pub use self::sketch::ThetaSketch; pub use self::sketch::ThetaSketchBuilder; diff --git a/datasketches/tests/theta_jaccard_similarity_test.rs b/datasketches/tests/theta_jaccard_similarity_test.rs index aa808d16..614cb5b2 100644 --- a/datasketches/tests/theta_jaccard_similarity_test.rs +++ b/datasketches/tests/theta_jaccard_similarity_test.rs @@ -17,13 +17,13 @@ #![cfg(feature = "theta")] -use datasketches::theta::ThetaJaccardSimilarity; +use datasketches::theta::JaccardSimilarity; use datasketches::theta::ThetaSketch; fn assert_jaccard_exact(actual: datasketches::theta::JaccardSimilarity, expected: f64) { - assert_eq!(actual.lower_bound, expected); - assert_eq!(actual.estimate, expected); - assert_eq!(actual.upper_bound, expected); + assert_eq!(actual.lower_bound(), expected); + assert_eq!(actual.estimate(), expected); + assert_eq!(actual.upper_bound(), expected); } fn assert_close(actual: f64, expected: f64, margin: f64) { @@ -54,7 +54,7 @@ fn test_empty() { let sketch_a = ThetaSketch::builder().build(); let sketch_b = ThetaSketch::builder().build(); - let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -63,11 +63,10 @@ fn test_empty() { fn test_same_sketch_exact_mode() { let sketch = sketch_with_range(0, 1000); - let jaccard = ThetaJaccardSimilarity::jaccard(&sketch, &sketch).unwrap(); + let jaccard = JaccardSimilarity::between(&sketch, &sketch).unwrap(); assert_jaccard_exact(jaccard, 1.0); - let jaccard = - ThetaJaccardSimilarity::jaccard(&sketch.compact(true), &sketch.compact(true)).unwrap(); + let jaccard = JaccardSimilarity::between(&sketch.compact(true), &sketch.compact(true)).unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -76,11 +75,11 @@ fn test_full_overlap_exact_mode() { let sketch_a = sketch_with_range(0, 1000); let sketch_b = sketch_with_range(0, 1000); - let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); let jaccard = - ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -89,11 +88,11 @@ fn test_disjoint_exact_mode() { let sketch_a = sketch_with_range(0, 1000); let sketch_b = sketch_with_range(1000, 1000); - let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); + let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 0.0); let jaccard = - ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); assert_jaccard_exact(jaccard, 0.0); } @@ -102,16 +101,16 @@ fn test_half_overlap_estimation_mode() { let sketch_a = sketch_with_range(0, 10000); let sketch_b = sketch_with_range(5000, 10000); - let jaccard = ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).unwrap(); - assert_close(jaccard.lower_bound, 0.33, 0.01); - assert_close(jaccard.estimate, 0.33, 0.01); - assert_close(jaccard.upper_bound, 0.33, 0.01); + let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); + assert_close(jaccard.lower_bound(), 0.33, 0.01); + assert_close(jaccard.estimate(), 0.33, 0.01); + assert_close(jaccard.upper_bound(), 0.33, 0.01); let jaccard = - ThetaJaccardSimilarity::jaccard(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); - assert_close(jaccard.lower_bound, 0.33, 0.01); - assert_close(jaccard.estimate, 0.33, 0.01); - assert_close(jaccard.upper_bound, 0.33, 0.01); + JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + assert_close(jaccard.lower_bound(), 0.33, 0.01); + assert_close(jaccard.estimate(), 0.33, 0.01); + assert_close(jaccard.upper_bound(), 0.33, 0.01); } #[test] @@ -120,20 +119,20 @@ fn test_half_overlap_estimation_mode_custom_seed() { let sketch_a = sketch_with_range_and_seed(0, 10000, seed); let sketch_b = sketch_with_range_and_seed(5000, 10000, seed); - let jaccard = ThetaJaccardSimilarity::jaccard_with_seed(&sketch_a, &sketch_b, seed).unwrap(); - assert_close(jaccard.lower_bound, 0.33, 0.01); - assert_close(jaccard.estimate, 0.33, 0.01); - assert_close(jaccard.upper_bound, 0.33, 0.01); + let jaccard = JaccardSimilarity::between_with_seed(&sketch_a, &sketch_b, seed).unwrap(); + assert_close(jaccard.lower_bound(), 0.33, 0.01); + assert_close(jaccard.estimate(), 0.33, 0.01); + assert_close(jaccard.upper_bound(), 0.33, 0.01); - let jaccard = ThetaJaccardSimilarity::jaccard_with_seed( + let jaccard = JaccardSimilarity::between_with_seed( &sketch_a.compact(true), &sketch_b.compact(true), seed, ) .unwrap(); - assert_close(jaccard.lower_bound, 0.33, 0.01); - assert_close(jaccard.estimate, 0.33, 0.01); - assert_close(jaccard.upper_bound, 0.33, 0.01); + assert_close(jaccard.lower_bound(), 0.33, 0.01); + assert_close(jaccard.estimate(), 0.33, 0.01); + assert_close(jaccard.upper_bound(), 0.33, 0.01); } #[test] @@ -143,5 +142,5 @@ fn test_seed_mismatch() { let mut sketch_b = ThetaSketch::builder().seed(123).build(); sketch_b.update(1u64); - assert!(ThetaJaccardSimilarity::jaccard(&sketch_a, &sketch_b).is_err()); + assert!(JaccardSimilarity::between(&sketch_a, &sketch_b).is_err()); } From 419be13d0e980ad8471105c59f9a9dc1c4be724c Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Sat, 1 Aug 2026 18:59:27 +0800 Subject: [PATCH 07/10] refactor(theta): share jaccard set operations --- .../src/thetacommon/jaccard_similarity.rs | 95 +++++++++++++++---- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/datasketches/src/thetacommon/jaccard_similarity.rs b/datasketches/src/thetacommon/jaccard_similarity.rs index cc24f03a..12e4929c 100644 --- a/datasketches/src/thetacommon/jaccard_similarity.rs +++ b/datasketches/src/thetacommon/jaccard_similarity.rs @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -use std::collections::BTreeSet; - use crate::common::ResizeFactor; use crate::error::Error; use crate::thetacommon::RetainedEntry; use crate::thetacommon::ThetaFamilySketchView; use crate::thetacommon::bounds_binomial_proportions; -use crate::thetacommon::constants::DEFAULT_LG_K; +use crate::thetacommon::constants::MAX_LG_K; use crate::thetacommon::constants::MAX_THETA; +use crate::thetacommon::constants::MIN_LG_K; +use crate::thetacommon::intersection::IntersectionMergePolicy; +use crate::thetacommon::intersection::IntersectionState; use crate::thetacommon::union::UnionMergePolicy; use crate::thetacommon::union::UnionState; @@ -37,12 +38,52 @@ pub(crate) struct RawThetaJaccardSimilarity { } #[derive(Debug)] -struct NoopUnionPolicy; +struct NoopMergePolicy; + +impl UnionMergePolicy for NoopMergePolicy { + fn merge(&self, _existing: &mut E, _incoming: E) {} +} -impl UnionMergePolicy for NoopUnionPolicy { +impl IntersectionMergePolicy for NoopMergePolicy { fn merge(&self, _existing: &mut E, _incoming: E) {} } +struct CompactSketchView { + entries: Vec, + theta: u64, + seed_hash: u16, + ordered: bool, + empty: bool, +} + +impl ThetaFamilySketchView for CompactSketchView { + type Entry = E; + + fn seed_hash(&self) -> u16 { + self.seed_hash + } + + fn theta64(&self) -> u64 { + self.theta + } + + fn is_empty(&self) -> bool { + self.empty + } + + fn is_ordered(&self) -> bool { + self.ordered + } + + fn iter(&self) -> impl Iterator + '_ { + self.entries.iter().cloned() + } + + fn num_retained(&self) -> usize { + self.entries.len() + } +} + impl RawThetaJaccardSimilarity { pub(crate) fn compute(sketch_a: &A, sketch_b: &B, seed: u64) -> Result where @@ -57,7 +98,13 @@ impl RawThetaJaccardSimilarity { return Ok(Self::exact(0.0)); } - let mut union = UnionState::new(DEFAULT_LG_K, ResizeFactor::X8, 1.0, seed, NoopUnionPolicy); + let mut union = UnionState::new( + union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()), + ResizeFactor::X8, + 1.0, + seed, + NoopMergePolicy, + ); union.update(sketch_a)?; union.update(sketch_b)?; let union = union.to_compact_parts(false); @@ -70,18 +117,24 @@ impl RawThetaJaccardSimilarity { return Ok(Self::exact(1.0)); } - let right_hashes: BTreeSet<_> = sketch_b - .iter() - .map(|entry| entry.hash()) - .filter(|hash| *hash < union.theta) - .collect(); - let intersection_count = sketch_a - .iter() - .map(|entry| entry.hash()) - .filter(|hash| *hash < union.theta && right_hashes.contains(hash)) - .count() as u64; - - Self::ratio_bounds(union.entries.len() as u64, intersection_count, union.theta) + let union = CompactSketchView { + entries: union.entries, + theta: union.theta, + seed_hash: union.seed_hash, + ordered: union.ordered, + empty: union.empty, + }; + let mut intersection = IntersectionState::new(seed, NoopMergePolicy); + intersection.update(sketch_a)?; + intersection.update(sketch_b)?; + intersection.update(&union)?; + let intersection = intersection.result(false); + + Self::ratio_bounds( + union.num_retained() as u64, + intersection.entries.len() as u64, + union.theta64(), + ) } fn exact(value: f64) -> Self { @@ -141,3 +194,9 @@ fn sampling_adjuster(sampling_probability: f64) -> f64 { adjustment + (0.01 * (sampling_probability - 0.5)) } } + +fn union_lg_k(left_count: usize, right_count: usize) -> u8 { + let required_capacity = left_count.saturating_add(right_count).max(1); + let lg_k = usize::BITS - (required_capacity - 1).leading_zeros(); + (lg_k as u8).clamp(MIN_LG_K, MAX_LG_K) +} From aeec2fd725e03f8992e2ea5119fdf5d78a96c368 Mon Sep 17 00:00:00 2001 From: Hawkingrei Date: Sun, 2 Aug 2026 15:36:06 +0800 Subject: [PATCH 08/10] refactor(theta): complete jaccard operator design --- datasketches/src/theta/jaccard_similarity.rs | 83 ++++--- datasketches/src/theta/mod.rs | 1 + .../bounds_binomial_proportions.rs | 31 ++- .../src/thetacommon/jaccard_similarity.rs | 235 ++++++++++++------ datasketches/src/thetacommon/mod.rs | 7 + datasketches/src/tuple/jaccard_similarity.rs | 83 +++++++ datasketches/src/tuple/mod.rs | 3 + datasketches/src/tuple/sketch.rs | 8 + .../tests/theta_test/jaccard_similarity.rs | 86 +++++-- .../tests/tuple_test/jaccard_similarity.rs | 138 ++++++++++ datasketches/tests/tuple_test/main.rs | 1 + 11 files changed, 533 insertions(+), 143 deletions(-) create mode 100644 datasketches/src/tuple/jaccard_similarity.rs create mode 100644 datasketches/tests/tuple_test/jaccard_similarity.rs diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index db4d86b3..d06ea8fc 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -20,55 +20,56 @@ use crate::error::Error; use crate::hash::DEFAULT_UPDATE_SEED; use crate::theta::ThetaSketchView; -use crate::thetacommon::jaccard_similarity::RawThetaJaccardSimilarity; +pub use crate::thetacommon::jaccard_similarity::JaccardSimilarity; +use crate::thetacommon::jaccard_similarity::JaccardSimilarityOperator; -/// Jaccard similarity result for two Theta sketches. +/// Jaccard similarity operator for Theta sketches. /// -/// The bounds use a 95.4% confidence interval, equivalent to +/- 2 standard deviations. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct JaccardSimilarity { - lower_bound: f64, - estimate: f64, - upper_bound: f64, +/// This is a stateless operator other than its configured hash seed. The returned +/// [`JaccardSimilarity`] contains the estimate and its 95.4% confidence interval. +/// +/// # Examples +/// +/// ``` +/// # use datasketches::theta::{ThetaJaccardSimilarity, ThetaSketchBuilder}; +/// let mut a = ThetaSketchBuilder::default().build(); +/// let mut b = ThetaSketchBuilder::default().build(); +/// a.update("apple"); +/// b.update("apple"); +/// +/// let result = ThetaJaccardSimilarity::default().compute(&a, &b).unwrap(); +/// assert_eq!(result.estimate(), 1.0); +/// ``` +#[derive(Clone, Copy, Debug)] +pub struct ThetaJaccardSimilarity { + op: JaccardSimilarityOperator, } -impl JaccardSimilarity { - /// Computes the Jaccard similarity index with the default update seed. - pub fn between( - sketch_a: &A, - sketch_b: &B, - ) -> Result { - Self::between_with_seed(sketch_a, sketch_b, DEFAULT_UPDATE_SEED) +impl Default for ThetaJaccardSimilarity { + fn default() -> Self { + Self::with_seed(DEFAULT_UPDATE_SEED) + } +} + +impl ThetaJaccardSimilarity { + /// Creates a Jaccard similarity operator for the given `seed`. + pub fn with_seed(seed: u64) -> Self { + Self { + op: JaccardSimilarityOperator::new(seed), + } } - /// Computes the Jaccard similarity index with an explicit update seed. + /// Computes the Jaccard similarity index for `sketch_a` and `sketch_b`. /// - /// Returns an error if a non-empty sketch was built with a different seed. - pub fn between_with_seed( + /// # Errors + /// + /// Returns an error if either non-empty sketch was built with a seed different from this + /// operator's configured seed. + pub fn compute( + &self, sketch_a: &A, sketch_b: &B, - seed: u64, - ) -> Result { - let raw = RawThetaJaccardSimilarity::compute(sketch_a, sketch_b, seed)?; - Ok(Self { - lower_bound: raw.lower_bound, - estimate: raw.estimate, - upper_bound: raw.upper_bound, - }) - } - - /// Returns the approximate lower bound for the Jaccard index. - pub fn lower_bound(&self) -> f64 { - self.lower_bound - } - - /// Returns the estimate of the Jaccard index. - pub fn estimate(&self) -> f64 { - self.estimate - } - - /// Returns the approximate upper bound for the Jaccard index. - pub fn upper_bound(&self) -> f64 { - self.upper_bound + ) -> Result { + self.op.compute(sketch_a, sketch_b) } } diff --git a/datasketches/src/theta/mod.rs b/datasketches/src/theta/mod.rs index 96d648a9..0b408aff 100644 --- a/datasketches/src/theta/mod.rs +++ b/datasketches/src/theta/mod.rs @@ -52,6 +52,7 @@ pub use self::a_not_b::ThetaANotB; pub use self::hash_table::ThetaEntry; pub use self::intersection::ThetaIntersection; pub use self::jaccard_similarity::JaccardSimilarity; +pub use self::jaccard_similarity::ThetaJaccardSimilarity; pub use self::sketch::CompactThetaSketch; pub use self::sketch::ThetaSketch; pub use self::sketch::ThetaSketchBuilder; diff --git a/datasketches/src/thetacommon/bounds_binomial_proportions.rs b/datasketches/src/thetacommon/bounds_binomial_proportions.rs index ba603a07..b075833f 100644 --- a/datasketches/src/thetacommon/bounds_binomial_proportions.rs +++ b/datasketches/src/thetacommon/bounds_binomial_proportions.rs @@ -116,8 +116,8 @@ fn abramowitz_stegun_formula_26p5p22(a: f64, b: f64, yp: f64) -> f64 { let b2m1 = (2.0 * b) - 1.0; let a2m1 = (2.0 * a) - 1.0; let lambda = ((yp * yp) - 3.0) / 6.0; - let htmp = (1.0 / a2m1) + (1.0 / b2m1); - let h = 2.0 / htmp; + let reciprocal_sum = (1.0 / a2m1) + (1.0 / b2m1); + let h = 2.0 / reciprocal_sum; let term1 = (yp * (h + lambda).sqrt()) / h; let term2 = (1.0 / b2m1) - (1.0 / a2m1); let term3 = (lambda + (5.0 / 6.0)) - (2.0 / (3.0 * h)); @@ -158,4 +158,31 @@ mod tests { assert_eq!(approximate_lower_bound_on_p(10, 0, 2.0).unwrap(), 0.0); assert_eq!(approximate_upper_bound_on_p(10, 10, 2.0).unwrap(), 1.0); } + + #[test] + fn matches_cross_language_reference_vectors() { + const LOWER: [f64; 6] = [ + 0.0, + 0.004592032688529923, + 0.04725537386564205, + 0.1396230607626959, + 0.2735831034867167, + 0.4692424353373485, + ]; + const UPPER: [f64; 6] = [ + 0.5307575646626514, + 0.7264168965132833, + 0.860376939237304, + 0.952744626134358, + 0.9954079673114701, + 1.0, + ]; + + for k in 0..=5 { + let lower = approximate_lower_bound_on_p(5, k, 2.0).unwrap(); + let upper = approximate_upper_bound_on_p(5, k, 2.0).unwrap(); + assert!((lower - LOWER[k as usize]).abs() < 1e-14); + assert!((upper - UPPER[k as usize]).abs() < 1e-14); + } + } } diff --git a/datasketches/src/thetacommon/jaccard_similarity.rs b/datasketches/src/thetacommon/jaccard_similarity.rs index 12e4929c..fe9def2d 100644 --- a/datasketches/src/thetacommon/jaccard_similarity.rs +++ b/datasketches/src/thetacommon/jaccard_similarity.rs @@ -30,14 +30,131 @@ use crate::thetacommon::union::UnionState; const NUM_STD_DEVS: f64 = 2.0; +/// Jaccard similarity estimate and confidence bounds for two Theta-family sketches. +/// +/// The bounds use a 95.4% confidence interval, equivalent to +/- 2 standard deviations. #[derive(Clone, Copy, Debug, PartialEq)] -pub(crate) struct RawThetaJaccardSimilarity { - pub(crate) lower_bound: f64, - pub(crate) estimate: f64, - pub(crate) upper_bound: f64, +pub struct JaccardSimilarity { + lower_bound: f64, + estimate: f64, + upper_bound: f64, } -#[derive(Debug)] +impl JaccardSimilarity { + /// Returns the approximate lower bound for the Jaccard index. + pub fn lower_bound(&self) -> f64 { + self.lower_bound + } + + /// Returns the estimate of the Jaccard index. + pub fn estimate(&self) -> f64 { + self.estimate + } + + /// Returns the approximate upper bound for the Jaccard index. + pub fn upper_bound(&self) -> f64 { + self.upper_bound + } + + fn exact(value: f64) -> Self { + Self { + lower_bound: value, + estimate: value, + upper_bound: value, + } + } + + fn ratio_bounds(union_count: u64, intersection_count: u64, theta: u64) -> Result { + if intersection_count > union_count { + return Err(Error::invalid_argument(format!( + "intersection count cannot exceed union count: {intersection_count} > {union_count}" + ))); + } + if union_count == 0 { + return Ok(Self { + lower_bound: 0.0, + estimate: 0.5, + upper_bound: 1.0, + }); + } + + let sampling_probability = theta as f64 / MAX_THETA as f64; + if sampling_probability <= 0.0 || sampling_probability > 1.0 { + return Err(Error::invalid_argument(format!( + "theta must produce a probability in (0.0, 1.0], got {sampling_probability}" + ))); + } + if sampling_probability == 1.0 { + return Ok(Self::exact(intersection_count as f64 / union_count as f64)); + } + + let adjustment = NUM_STD_DEVS * sampling_adjuster(sampling_probability); + Ok(Self { + lower_bound: bounds_binomial_proportions::approximate_lower_bound_on_p( + union_count, + intersection_count, + adjustment, + )?, + estimate: intersection_count as f64 / union_count as f64, + upper_bound: bounds_binomial_proportions::approximate_upper_bound_on_p( + union_count, + intersection_count, + adjustment, + )?, + }) + } +} + +#[derive(Clone, Copy, Debug)] +struct KeyEntry { + hash: u64, +} + +impl RetainedEntry for KeyEntry { + fn hash(&self) -> u64 { + self.hash + } +} + +struct KeySketchView<'a, S> { + sketch: &'a S, +} + +impl<'a, S> KeySketchView<'a, S> { + fn new(sketch: &'a S) -> Self { + Self { sketch } + } +} + +impl ThetaFamilySketchView for KeySketchView<'_, S> { + type Entry = KeyEntry; + + fn seed_hash(&self) -> u16 { + self.sketch.seed_hash() + } + + fn theta64(&self) -> u64 { + self.sketch.theta64() + } + + fn is_empty(&self) -> bool { + self.sketch.is_empty() + } + + fn is_ordered(&self) -> bool { + self.sketch.is_ordered() + } + + fn iter(&self) -> impl Iterator + '_ { + self.sketch.iter_hashes().map(|hash| KeyEntry { hash }) + } + + fn num_retained(&self) -> usize { + self.sketch.num_retained() + } +} + +#[derive(Clone, Copy, Debug)] struct NoopMergePolicy; impl UnionMergePolicy for NoopMergePolicy { @@ -48,16 +165,16 @@ impl IntersectionMergePolicy for NoopMergePolicy { fn merge(&self, _existing: &mut E, _incoming: E) {} } -struct CompactSketchView { - entries: Vec, +struct CompactKeySketchView { + entries: Vec, theta: u64, seed_hash: u16, ordered: bool, empty: bool, } -impl ThetaFamilySketchView for CompactSketchView { - type Entry = E; +impl ThetaFamilySketchView for CompactKeySketchView { + type Entry = KeyEntry; fn seed_hash(&self) -> u16 { self.seed_hash @@ -75,8 +192,8 @@ impl ThetaFamilySketchView for CompactSketchView { self.ordered } - fn iter(&self) -> impl Iterator + '_ { - self.entries.iter().cloned() + fn iter(&self) -> impl Iterator + '_ { + self.entries.iter().copied() } fn num_retained(&self) -> usize { @@ -84,106 +201,74 @@ impl ThetaFamilySketchView for CompactSketchView { } } -impl RawThetaJaccardSimilarity { - pub(crate) fn compute(sketch_a: &A, sketch_b: &B, seed: u64) -> Result +/// Configured Jaccard operator shared by Theta and Tuple public wrappers. +#[derive(Clone, Copy, Debug)] +pub(crate) struct JaccardSimilarityOperator { + seed: u64, +} + +impl JaccardSimilarityOperator { + pub(crate) fn new(seed: u64) -> Self { + Self { seed } + } + + pub(crate) fn compute( + &self, + sketch_a: &A, + sketch_b: &B, + ) -> Result where A: ThetaFamilySketchView, - B: ThetaFamilySketchView, - A::Entry: Clone, + B: ThetaFamilySketchView, { if sketch_a.is_empty() && sketch_b.is_empty() { - return Ok(Self::exact(1.0)); + return Ok(JaccardSimilarity::exact(1.0)); } if sketch_a.is_empty() || sketch_b.is_empty() { - return Ok(Self::exact(0.0)); + return Ok(JaccardSimilarity::exact(0.0)); } + let sketch_a = KeySketchView::new(sketch_a); + let sketch_b = KeySketchView::new(sketch_b); let mut union = UnionState::new( union_lg_k(sketch_a.num_retained(), sketch_b.num_retained()), ResizeFactor::X8, 1.0, - seed, + self.seed, NoopMergePolicy, ); - union.update(sketch_a)?; - union.update(sketch_b)?; + union.update(&sketch_a)?; + union.update(&sketch_b)?; let union = union.to_compact_parts(false); - if union.entries.len() == sketch_a.num_retained() + if !union.entries.is_empty() + && union.entries.len() == sketch_a.num_retained() && union.entries.len() == sketch_b.num_retained() && union.theta == sketch_a.theta64() && union.theta == sketch_b.theta64() { - return Ok(Self::exact(1.0)); + return Ok(JaccardSimilarity::exact(1.0)); } - let union = CompactSketchView { + let union = CompactKeySketchView { entries: union.entries, theta: union.theta, seed_hash: union.seed_hash, ordered: union.ordered, empty: union.empty, }; - let mut intersection = IntersectionState::new(seed, NoopMergePolicy); - intersection.update(sketch_a)?; - intersection.update(sketch_b)?; + let mut intersection = IntersectionState::new(self.seed, NoopMergePolicy); + intersection.update(&sketch_a)?; + intersection.update(&sketch_b)?; intersection.update(&union)?; let intersection = intersection.result(false); - Self::ratio_bounds( + JaccardSimilarity::ratio_bounds( union.num_retained() as u64, intersection.entries.len() as u64, union.theta64(), ) } - - fn exact(value: f64) -> Self { - Self { - lower_bound: value, - estimate: value, - upper_bound: value, - } - } - - fn ratio_bounds(union_count: u64, intersection_count: u64, theta: u64) -> Result { - if intersection_count > union_count { - return Err(Error::invalid_argument(format!( - "intersection count cannot exceed union count: {intersection_count} > {union_count}" - ))); - } - if union_count == 0 { - return Ok(Self { - lower_bound: 0.0, - estimate: 0.5, - upper_bound: 1.0, - }); - } - - let sampling_probability = theta as f64 / MAX_THETA as f64; - if sampling_probability <= 0.0 || sampling_probability > 1.0 { - return Err(Error::invalid_argument(format!( - "theta must produce a probability in (0.0, 1.0], got {sampling_probability}" - ))); - } - if sampling_probability == 1.0 { - return Ok(Self::exact(intersection_count as f64 / union_count as f64)); - } - - let adjustment = NUM_STD_DEVS * sampling_adjuster(sampling_probability); - Ok(Self { - lower_bound: bounds_binomial_proportions::approximate_lower_bound_on_p( - union_count, - intersection_count, - adjustment, - )?, - estimate: intersection_count as f64 / union_count as f64, - upper_bound: bounds_binomial_proportions::approximate_upper_bound_on_p( - union_count, - intersection_count, - adjustment, - )?, - }) - } } fn sampling_adjuster(sampling_probability: f64) -> f64 { diff --git a/datasketches/src/thetacommon/mod.rs b/datasketches/src/thetacommon/mod.rs index 6b666f39..7e8c87ff 100644 --- a/datasketches/src/thetacommon/mod.rs +++ b/datasketches/src/thetacommon/mod.rs @@ -55,6 +55,13 @@ pub trait ThetaFamilySketchView { /// Return an iterator over retained entries. fn iter(&self) -> impl Iterator + '_; + /// Return an iterator over retained hash keys without requiring callers to inspect payloads. + /// + /// Tuple sketches override this method so key-only operations do not clone summary values. + fn iter_hashes(&self) -> impl Iterator + '_ { + self.iter().map(|entry| entry.hash()) + } + /// Return the number of retained entries. fn num_retained(&self) -> usize; } diff --git a/datasketches/src/tuple/jaccard_similarity.rs b/datasketches/src/tuple/jaccard_similarity.rs new file mode 100644 index 00000000..7017119a --- /dev/null +++ b/datasketches/src/tuple/jaccard_similarity.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Jaccard similarity for Tuple sketches. + +use crate::error::Error; +use crate::hash::DEFAULT_UPDATE_SEED; +pub use crate::thetacommon::jaccard_similarity::JaccardSimilarity; +use crate::thetacommon::jaccard_similarity::JaccardSimilarityOperator; +use crate::tuple::TupleSketchView; + +/// Jaccard similarity operator for Tuple sketches. +/// +/// Only retained hash keys participate in the similarity calculation. Summary values are ignored, +/// so the two inputs may use different summary types. The returned [`JaccardSimilarity`] contains +/// the estimate and its 95.4% confidence interval. +/// +/// # Examples +/// +/// ``` +/// # use datasketches::tuple::{DefaultUpdatePolicy, TupleJaccardSimilarity, TupleSketchBuilder}; +/// let policy = DefaultUpdatePolicy::::default(); +/// let mut a = TupleSketchBuilder::new(policy).build(); +/// let mut b = TupleSketchBuilder::new(policy).build(); +/// a.update("apple", 1); +/// b.update("apple", 2); +/// +/// let result = TupleJaccardSimilarity::default().compute(&a, &b).unwrap(); +/// assert_eq!(result.estimate(), 1.0); +/// ``` +#[derive(Clone, Copy, Debug)] +pub struct TupleJaccardSimilarity { + op: JaccardSimilarityOperator, +} + +impl Default for TupleJaccardSimilarity { + fn default() -> Self { + Self::with_seed(DEFAULT_UPDATE_SEED) + } +} + +impl TupleJaccardSimilarity { + /// Creates a Jaccard similarity operator for the given `seed`. + pub fn with_seed(seed: u64) -> Self { + Self { + op: JaccardSimilarityOperator::new(seed), + } + } + + /// Computes the Jaccard similarity index for `sketch_a` and `sketch_b`. + /// + /// Summary values do not participate in the comparison. + /// + /// # Errors + /// + /// Returns an error if either non-empty sketch was built with a seed different from this + /// operator's configured seed. + pub fn compute( + &self, + sketch_a: &A, + sketch_b: &B, + ) -> Result + where + A: TupleSketchView, + B: TupleSketchView, + { + self.op.compute(sketch_a, sketch_b) + } +} diff --git a/datasketches/src/tuple/mod.rs b/datasketches/src/tuple/mod.rs index 4c880586..8dba57d7 100644 --- a/datasketches/src/tuple/mod.rs +++ b/datasketches/src/tuple/mod.rs @@ -41,6 +41,7 @@ mod a_not_b; mod hash_table; mod intersection; +mod jaccard_similarity; mod policy; mod serialization; mod sketch; @@ -49,6 +50,8 @@ mod union; pub use self::a_not_b::TupleANotB; pub use self::hash_table::TupleEntry; pub use self::intersection::TupleIntersection; +pub use self::jaccard_similarity::JaccardSimilarity; +pub use self::jaccard_similarity::TupleJaccardSimilarity; pub use self::policy::DefaultUnionPolicy; pub use self::policy::DefaultUpdatePolicy; pub use self::policy::SummaryCombinePolicy; diff --git a/datasketches/src/tuple/sketch.rs b/datasketches/src/tuple/sketch.rs index b06c4067..ef3dda44 100644 --- a/datasketches/src/tuple/sketch.rs +++ b/datasketches/src/tuple/sketch.rs @@ -277,6 +277,10 @@ where .map(|(hash, summary)| TupleEntry::new(hash, summary.clone())) } + fn iter_hashes(&self) -> impl Iterator + '_ { + self.table.iter().map(|(hash, _summary)| hash) + } + fn num_retained(&self) -> usize { self.table.num_retained() } @@ -591,6 +595,10 @@ impl ThetaFamilySketchView for CompactTupleSketch { self.entries.iter().cloned() } + fn iter_hashes(&self) -> impl Iterator + '_ { + self.entries.iter().map(TupleEntry::hash) + } + fn num_retained(&self) -> usize { self.entries.len() } diff --git a/datasketches/tests/theta_test/jaccard_similarity.rs b/datasketches/tests/theta_test/jaccard_similarity.rs index 6e300aba..2af60808 100644 --- a/datasketches/tests/theta_test/jaccard_similarity.rs +++ b/datasketches/tests/theta_test/jaccard_similarity.rs @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -#![cfg(feature = "theta")] - use datasketches::theta::JaccardSimilarity; +use datasketches::theta::ThetaJaccardSimilarity; use datasketches::theta::ThetaSketch; use datasketches::theta::ThetaSketchBuilder; -fn assert_jaccard_exact(actual: datasketches::theta::JaccardSimilarity, expected: f64) { +fn assert_jaccard_exact(actual: JaccardSimilarity, expected: f64) { assert_eq!(actual.lower_bound(), expected); assert_eq!(actual.estimate(), expected); assert_eq!(actual.upper_bound(), expected); @@ -35,9 +34,9 @@ fn assert_close(actual: f64, expected: f64, margin: f64) { } fn assert_jaccard_estimate(actual: JaccardSimilarity, expected: f64) { + assert_close(actual.lower_bound(), expected, 0.01); assert_close(actual.estimate(), expected, 0.01); - assert!(actual.lower_bound() <= actual.estimate()); - assert!(actual.estimate() <= actual.upper_bound()); + assert_close(actual.upper_bound(), expected, 0.01); } fn sketch_with_range(start: u64, count: u64) -> ThetaSketch { @@ -61,7 +60,9 @@ fn test_empty() { let sketch_a = ThetaSketchBuilder::default().build(); let sketch_b = ThetaSketchBuilder::default().build(); - let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); + let jaccard = ThetaJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -70,10 +71,13 @@ fn test_empty() { fn test_same_sketch_exact_mode() { let sketch = sketch_with_range(0, 1000); - let jaccard = JaccardSimilarity::between(&sketch, &sketch).unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch, &sketch).unwrap(); assert_jaccard_exact(jaccard, 1.0); - let jaccard = JaccardSimilarity::between(&sketch.compact(true), &sketch.compact(true)).unwrap(); + let jaccard = operator + .compute(&sketch.compact(true), &sketch.compact(true)) + .unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -82,11 +86,13 @@ fn test_full_overlap_exact_mode() { let sketch_a = sketch_with_range(0, 1000); let sketch_b = sketch_with_range(0, 1000); - let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 1.0); - let jaccard = - JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); assert_jaccard_exact(jaccard, 1.0); } @@ -95,11 +101,13 @@ fn test_disjoint_exact_mode() { let sketch_a = sketch_with_range(0, 1000); let sketch_b = sketch_with_range(1000, 1000); - let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_exact(jaccard, 0.0); - let jaccard = - JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); assert_jaccard_exact(jaccard, 0.0); } @@ -108,11 +116,13 @@ fn test_half_overlap_estimation_mode() { let sketch_a = sketch_with_range(0, 10000); let sketch_b = sketch_with_range(5000, 10000); - let jaccard = JaccardSimilarity::between(&sketch_a, &sketch_b).unwrap(); + let operator = ThetaJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_estimate(jaccard, 0.33); - let jaccard = - JaccardSimilarity::between(&sketch_a.compact(true), &sketch_b.compact(true)).unwrap(); + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); assert_jaccard_estimate(jaccard, 0.33); } @@ -122,15 +132,13 @@ fn test_half_overlap_estimation_mode_custom_seed() { let sketch_a = sketch_with_range_and_seed(0, 10000, seed); let sketch_b = sketch_with_range_and_seed(5000, 10000, seed); - let jaccard = JaccardSimilarity::between_with_seed(&sketch_a, &sketch_b, seed).unwrap(); + let operator = ThetaJaccardSimilarity::with_seed(seed); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); assert_jaccard_estimate(jaccard, 0.33); - let jaccard = JaccardSimilarity::between_with_seed( - &sketch_a.compact(true), - &sketch_b.compact(true), - seed, - ) - .unwrap(); + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); assert_jaccard_estimate(jaccard, 0.33); } @@ -141,5 +149,33 @@ fn test_seed_mismatch() { let mut sketch_b = ThetaSketchBuilder::default().seed(123).build(); sketch_b.update(1u64); - assert!(JaccardSimilarity::between(&sketch_a, &sketch_b).is_err()); + assert!( + ThetaJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .is_err() + ); +} + +#[test] +fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() { + let mut sketch_a = ThetaSketchBuilder::default() + .sampling_probability(1e-12) + .build(); + let mut sketch_b = ThetaSketchBuilder::default() + .sampling_probability(1e-12) + .build(); + sketch_a.update("apple"); + sketch_b.update("banana"); + + assert!(!sketch_a.is_empty()); + assert!(!sketch_b.is_empty()); + assert_eq!(sketch_a.num_retained(), 0); + assert_eq!(sketch_b.num_retained(), 0); + + let jaccard = ThetaJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .unwrap(); + assert_eq!(jaccard.lower_bound(), 0.0); + assert_eq!(jaccard.estimate(), 0.5); + assert_eq!(jaccard.upper_bound(), 1.0); } diff --git a/datasketches/tests/tuple_test/jaccard_similarity.rs b/datasketches/tests/tuple_test/jaccard_similarity.rs new file mode 100644 index 00000000..524504ba --- /dev/null +++ b/datasketches/tests/tuple_test/jaccard_similarity.rs @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::tuple::DefaultUpdatePolicy; +use datasketches::tuple::JaccardSimilarity; +use datasketches::tuple::TupleJaccardSimilarity; +use datasketches::tuple::TupleSketchBuilder; + +use crate::default_tuple_sketch_builder; +use crate::tuple_sketch_with_range; + +fn assert_jaccard_exact(actual: JaccardSimilarity, expected: f64) { + assert_eq!(actual.lower_bound(), expected); + assert_eq!(actual.estimate(), expected); + assert_eq!(actual.upper_bound(), expected); +} + +fn assert_close(actual: f64, expected: f64, margin: f64) { + assert!( + (actual - expected).abs() <= margin, + "actual={actual}, expected={expected}, margin={margin}" + ); +} + +fn assert_jaccard_estimate(actual: JaccardSimilarity, expected: f64) { + assert_close(actual.lower_bound(), expected, 0.01); + assert_close(actual.estimate(), expected, 0.01); + assert_close(actual.upper_bound(), expected, 0.01); +} + +#[test] +fn test_empty() { + let sketch_a = default_tuple_sketch_builder().build(); + let sketch_b = default_tuple_sketch_builder().build(); + + let jaccard = TupleJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .unwrap(); + + assert_jaccard_exact(jaccard, 1.0); +} + +#[test] +fn test_summary_values_and_types_do_not_affect_similarity() { + let mut sketch_a = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()).build(); + let mut sketch_b = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()).build(); + for key in 0..1000 { + sketch_a.update(key, 1u64); + sketch_b.update(key, -7i64); + } + + let operator = TupleJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); + assert_jaccard_exact(jaccard, 1.0); + + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); + assert_jaccard_exact(jaccard, 1.0); +} + +#[test] +fn test_half_overlap_estimation_mode() { + let sketch_a = tuple_sketch_with_range(0, 10000); + let sketch_b = tuple_sketch_with_range(5000, 10000); + + let operator = TupleJaccardSimilarity::default(); + let jaccard = operator.compute(&sketch_a, &sketch_b).unwrap(); + assert_jaccard_estimate(jaccard, 0.33); + + let jaccard = operator + .compute(&sketch_a.compact(true), &sketch_b.compact(true)) + .unwrap(); + assert_jaccard_estimate(jaccard, 0.33); +} + +#[test] +fn test_custom_seed_and_seed_mismatch() { + let seed = 123; + let mut sketch_a = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(seed) + .build(); + let mut sketch_b = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(seed) + .build(); + for value in 0..1000 { + sketch_a.update(value, 1u64); + sketch_b.update(value, 2u64); + } + + let jaccard = TupleJaccardSimilarity::with_seed(seed) + .compute(&sketch_a, &sketch_b) + .unwrap(); + assert_jaccard_exact(jaccard, 1.0); + assert!( + TupleJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .is_err() + ); +} + +#[test] +fn test_distinct_non_empty_sketches_with_no_retained_entries_are_uncertain() { + let mut sketch_a = default_tuple_sketch_builder() + .sampling_probability(1e-12) + .build(); + let mut sketch_b = default_tuple_sketch_builder() + .sampling_probability(1e-12) + .build(); + sketch_a.update("apple", 1u64); + sketch_b.update("banana", 1u64); + + assert!(!sketch_a.is_empty()); + assert!(!sketch_b.is_empty()); + assert_eq!(sketch_a.num_retained(), 0); + assert_eq!(sketch_b.num_retained(), 0); + + let jaccard = TupleJaccardSimilarity::default() + .compute(&sketch_a, &sketch_b) + .unwrap(); + assert_eq!(jaccard.lower_bound(), 0.0); + assert_eq!(jaccard.estimate(), 0.5); + assert_eq!(jaccard.upper_bound(), 1.0); +} diff --git a/datasketches/tests/tuple_test/main.rs b/datasketches/tests/tuple_test/main.rs index 887a2803..cbb6290a 100644 --- a/datasketches/tests/tuple_test/main.rs +++ b/datasketches/tests/tuple_test/main.rs @@ -17,6 +17,7 @@ mod a_not_b; mod intersection; +mod jaccard_similarity; mod sketch; mod union; From 18ad4b503b528284d098d4c369f61125257691fc Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 2 Aug 2026 17:00:54 +0800 Subject: [PATCH 09/10] fixup Signed-off-by: tison --- datasketches/src/theta/jaccard_similarity.rs | 2 +- datasketches/src/theta/mod.rs | 1 - .../src/thetacommon/jaccard_similarity.rs | 17 +++++++++++++++++ datasketches/src/thetacommon/mod.rs | 2 ++ datasketches/src/tuple/jaccard_similarity.rs | 2 +- datasketches/src/tuple/mod.rs | 1 - datasketches/src/tuple/sketch.rs | 8 -------- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/datasketches/src/theta/jaccard_similarity.rs b/datasketches/src/theta/jaccard_similarity.rs index d06ea8fc..0c1937f8 100644 --- a/datasketches/src/theta/jaccard_similarity.rs +++ b/datasketches/src/theta/jaccard_similarity.rs @@ -20,7 +20,7 @@ use crate::error::Error; use crate::hash::DEFAULT_UPDATE_SEED; use crate::theta::ThetaSketchView; -pub use crate::thetacommon::jaccard_similarity::JaccardSimilarity; +use crate::thetacommon::jaccard_similarity::JaccardSimilarity; use crate::thetacommon::jaccard_similarity::JaccardSimilarityOperator; /// Jaccard similarity operator for Theta sketches. diff --git a/datasketches/src/theta/mod.rs b/datasketches/src/theta/mod.rs index 0b408aff..87f21fe6 100644 --- a/datasketches/src/theta/mod.rs +++ b/datasketches/src/theta/mod.rs @@ -51,7 +51,6 @@ mod union; pub use self::a_not_b::ThetaANotB; pub use self::hash_table::ThetaEntry; pub use self::intersection::ThetaIntersection; -pub use self::jaccard_similarity::JaccardSimilarity; pub use self::jaccard_similarity::ThetaJaccardSimilarity; pub use self::sketch::CompactThetaSketch; pub use self::sketch::ThetaSketch; diff --git a/datasketches/src/thetacommon/jaccard_similarity.rs b/datasketches/src/thetacommon/jaccard_similarity.rs index fe9def2d..f8c2522a 100644 --- a/datasketches/src/thetacommon/jaccard_similarity.rs +++ b/datasketches/src/thetacommon/jaccard_similarity.rs @@ -17,6 +17,7 @@ use crate::common::ResizeFactor; use crate::error::Error; +use crate::hash::compute_seed_hash; use crate::thetacommon::RetainedEntry; use crate::thetacommon::ThetaFamilySketchView; use crate::thetacommon::bounds_binomial_proportions; @@ -228,6 +229,22 @@ impl JaccardSimilarityOperator { return Ok(JaccardSimilarity::exact(0.0)); } + let seed_hash = compute_seed_hash(self.seed); + if seed_hash != sketch_a.seed_hash() { + return Err(Error::invalid_argument(format!( + "incompatible seed hash: expected {}, got {}", + seed_hash, + sketch_a.seed_hash(), + ))); + } + if seed_hash != sketch_b.seed_hash() { + return Err(Error::invalid_argument(format!( + "incompatible seed hash: expected {}, got {}", + seed_hash, + sketch_b.seed_hash(), + ))); + } + let sketch_a = KeySketchView::new(sketch_a); let sketch_b = KeySketchView::new(sketch_b); let mut union = UnionState::new( diff --git a/datasketches/src/thetacommon/mod.rs b/datasketches/src/thetacommon/mod.rs index 7e8c87ff..6da798d4 100644 --- a/datasketches/src/thetacommon/mod.rs +++ b/datasketches/src/thetacommon/mod.rs @@ -26,6 +26,8 @@ pub(crate) mod intersection; pub(crate) mod jaccard_similarity; pub(crate) mod union; +pub use self::jaccard_similarity::JaccardSimilarity; + /// An entry retained by a Theta sketch family hash table. pub trait RetainedEntry { /// Return the hash used as this entry's key. diff --git a/datasketches/src/tuple/jaccard_similarity.rs b/datasketches/src/tuple/jaccard_similarity.rs index 7017119a..d9faa2d6 100644 --- a/datasketches/src/tuple/jaccard_similarity.rs +++ b/datasketches/src/tuple/jaccard_similarity.rs @@ -19,7 +19,7 @@ use crate::error::Error; use crate::hash::DEFAULT_UPDATE_SEED; -pub use crate::thetacommon::jaccard_similarity::JaccardSimilarity; +use crate::thetacommon::jaccard_similarity::JaccardSimilarity; use crate::thetacommon::jaccard_similarity::JaccardSimilarityOperator; use crate::tuple::TupleSketchView; diff --git a/datasketches/src/tuple/mod.rs b/datasketches/src/tuple/mod.rs index 8dba57d7..1ba8b58c 100644 --- a/datasketches/src/tuple/mod.rs +++ b/datasketches/src/tuple/mod.rs @@ -50,7 +50,6 @@ mod union; pub use self::a_not_b::TupleANotB; pub use self::hash_table::TupleEntry; pub use self::intersection::TupleIntersection; -pub use self::jaccard_similarity::JaccardSimilarity; pub use self::jaccard_similarity::TupleJaccardSimilarity; pub use self::policy::DefaultUnionPolicy; pub use self::policy::DefaultUpdatePolicy; diff --git a/datasketches/src/tuple/sketch.rs b/datasketches/src/tuple/sketch.rs index ef3dda44..b06c4067 100644 --- a/datasketches/src/tuple/sketch.rs +++ b/datasketches/src/tuple/sketch.rs @@ -277,10 +277,6 @@ where .map(|(hash, summary)| TupleEntry::new(hash, summary.clone())) } - fn iter_hashes(&self) -> impl Iterator + '_ { - self.table.iter().map(|(hash, _summary)| hash) - } - fn num_retained(&self) -> usize { self.table.num_retained() } @@ -595,10 +591,6 @@ impl ThetaFamilySketchView for CompactTupleSketch { self.entries.iter().cloned() } - fn iter_hashes(&self) -> impl Iterator + '_ { - self.entries.iter().map(TupleEntry::hash) - } - fn num_retained(&self) -> usize { self.entries.len() } From 373cfa67cc04110949780c00fe63530aec947a82 Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 2 Aug 2026 17:01:32 +0800 Subject: [PATCH 10/10] fixup Signed-off-by: tison --- datasketches/tests/theta_test/jaccard_similarity.rs | 2 +- datasketches/tests/tuple_test/jaccard_similarity.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasketches/tests/theta_test/jaccard_similarity.rs b/datasketches/tests/theta_test/jaccard_similarity.rs index 2af60808..428989f5 100644 --- a/datasketches/tests/theta_test/jaccard_similarity.rs +++ b/datasketches/tests/theta_test/jaccard_similarity.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use datasketches::theta::JaccardSimilarity; use datasketches::theta::ThetaJaccardSimilarity; use datasketches::theta::ThetaSketch; use datasketches::theta::ThetaSketchBuilder; +use datasketches::thetacommon::JaccardSimilarity; fn assert_jaccard_exact(actual: JaccardSimilarity, expected: f64) { assert_eq!(actual.lower_bound(), expected); diff --git a/datasketches/tests/tuple_test/jaccard_similarity.rs b/datasketches/tests/tuple_test/jaccard_similarity.rs index 524504ba..d5682850 100644 --- a/datasketches/tests/tuple_test/jaccard_similarity.rs +++ b/datasketches/tests/tuple_test/jaccard_similarity.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. +use datasketches::thetacommon::JaccardSimilarity; use datasketches::tuple::DefaultUpdatePolicy; -use datasketches::tuple::JaccardSimilarity; use datasketches::tuple::TupleJaccardSimilarity; use datasketches::tuple::TupleSketchBuilder;