From d79fafe548f191d5a7cace01fb4df31399204122 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 16 Nov 2026 13:25:44 -0800 Subject: [PATCH 1/2] EBPPS: keep the larger maximum weight after a merge internalMerge() computed the maximum weight of the two sketches and used it for every rho step, but never stored it. After a merge, a later update computed rho with the target's own, smaller maximum, so the sample stopped being proportional to weight, and toByteArray() wrote the stale value. The swap path in merge() copied the same stale value. This is the Java counterpart of apache/datasketches-cpp#581, and the new test is a port of the C++ one. Also correct the profile name in a UtilityIO javadoc example (check_cpp_files, not check-cpp-files) and add its missing closing quote. Co-Authored-By: Claude Opus 5.5 --- .../sampling/EbppsItemsSketch.java | 1 + .../apache/datasketches/common/UtilityIO.java | 2 +- .../sampling/EbppsItemsSketchTest.java | 61 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java index d1d147021..43d41f680 100644 --- a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java +++ b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java @@ -345,6 +345,7 @@ private void internalMerge(final EbppsItemsSketch other) { // avoid numeric issues by setting cumulative weight to the // pre-computed value cumulativeWt_ = finalCumWt; + wtMax_ = newWtMax; n_ = newN; } diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index f74f58824..f428e6dba 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -229,7 +229,7 @@ static void putBytesToFile(final Path basePath, final String fileName, final byt * Resolves the target directory for test data reading/writing. * The default is the project root, but it can be overridden like this: * - *

mvn clean test -Pcheck-cpp-files -Dtest.data.root="/tmp/custom_path

+ *

mvn clean test -Pcheck_cpp_files -Dtest.data.root="/tmp/custom_path"

* *

Calling the binary test jar: see src/test/resources/testng.xml

* diff --git a/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java b/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java index 97fdc1601..ef75dbc20 100644 --- a/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java +++ b/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java @@ -324,4 +324,65 @@ public void deserializeTooShort() { final MemorySegment shortSeg = seg.asSlice(0, seg.byteSize() - 1); EbppsItemsSketch.heapify(shortSeg, new ArrayOfLongsSerDe()); } + + // A merge must keep the larger maximum weight of the two sketches. Otherwise a later update + // uses the smaller one and the sample stops being proportional to weight. + // C depends only on the sequence of rho values, so these checks are deterministic. + // See apache/datasketches-cpp#581. + @Test + public void mergeKeepsMaxWeight() { + final int k = 10; + final int n = 1000; + final double heavyWt = 500.0; + + // reference: the same items in a single sketch + final EbppsItemsSketch ref = new EbppsItemsSketch<>(k); + for (long i = 0; i < n; ++i) { ref.update(i, 1.0); } + ref.update(-1L, heavyWt); + ref.update(-2L, 1.0); + assertEquals(ref.getC(), (n + 1 + heavyWt) / heavyWt, EPS); + + // the heavy item merged into the larger sketch + final EbppsItemsSketch big = buildUnitWeightSketch(k, n); + big.merge(buildHeavySketch(k, heavyWt)); + checkAfterOneUpdate(big, ref); + + // the larger sketch merged into the heavy one: merge() takes the swap path + final EbppsItemsSketch heavy = buildHeavySketch(k, heavyWt); + heavy.merge(buildUnitWeightSketch(k, n)); + checkAfterOneUpdate(heavy, ref); + + // the maximum weight must also survive serialization + final EbppsItemsSketch big2 = buildUnitWeightSketch(k, n); + big2.merge(buildHeavySketch(k, heavyWt)); + checkAfterOneUpdate(heapifyCopy(big2), ref); + + final EbppsItemsSketch heavy2 = buildHeavySketch(k, heavyWt); + heavy2.merge(buildUnitWeightSketch(k, n)); + checkAfterOneUpdate(heapifyCopy(heavy2), ref); + } + + private static EbppsItemsSketch buildUnitWeightSketch(final int k, final int n) { + final EbppsItemsSketch sk = new EbppsItemsSketch<>(k); + for (long i = 0; i < n; ++i) { sk.update(i, 1.0); } + return sk; + } + + private static EbppsItemsSketch buildHeavySketch(final int k, final double heavyWt) { + final EbppsItemsSketch sk = new EbppsItemsSketch<>(k); + sk.update(-1L, heavyWt); + return sk; + } + + private static EbppsItemsSketch heapifyCopy(final EbppsItemsSketch sk) { + final byte[] bytes = sk.toByteArray(new ArrayOfLongsSerDe()); + return EbppsItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + } + + private static void checkAfterOneUpdate(final EbppsItemsSketch sk, final EbppsItemsSketch ref) { + sk.update(-2L, 1.0); + assertEquals(sk.getN(), ref.getN()); + assertEquals(sk.getCumulativeWeight(), ref.getCumulativeWeight(), EPS); + assertEquals(sk.getC(), ref.getC(), EPS); + } } From d41dc7c14d508cccdc8689d9108aad1667140ab0 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 16 Nov 2026 13:51:36 -0800 Subject: [PATCH 2/2] UtilityIO: fix a javadoc link to the Java base path The link pointed to a javaPath member that does not exist; it now links to getJavaPath(). Co-Authored-By: Claude Opus 5.5 --- src/test/java/org/apache/datasketches/common/UtilityIO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index f428e6dba..c129ac2c4 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -190,7 +190,7 @@ public static byte[] getTestResourceBytes(final String fileName) { /** * Puts all the bytes of the given byte array to a file with the given fileName. - * This assumes that the base directory path is {@link #javaPath javaPath}. + * This assumes that the base directory path is {@link #getJavaPath()}. * @param fileName the name of the target file * @param bytes the given byte array */