diff --git a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java index d1d147021..43d41f680 100644 --- a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java +++ b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java @@ -345,6 +345,7 @@ private void internalMerge(final EbppsItemsSketch other) { // avoid numeric issues by setting cumulative weight to the // pre-computed value cumulativeWt_ = finalCumWt; + wtMax_ = newWtMax; n_ = newN; } diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index f74f58824..c129ac2c4 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -190,7 +190,7 @@ public static byte[] getTestResourceBytes(final String fileName) { /** * Puts all the bytes of the given byte array to a file with the given fileName. - * This assumes that the base directory path is {@link #javaPath javaPath}. + * This assumes that the base directory path is {@link #getJavaPath()}. * @param fileName the name of the target file * @param bytes the given byte array */ @@ -229,7 +229,7 @@ static void putBytesToFile(final Path basePath, final String fileName, final byt * Resolves the target directory for test data reading/writing. * The default is the project root, but it can be overridden like this: * - *

mvn clean test -Pcheck-cpp-files -Dtest.data.root="/tmp/custom_path

+ *

mvn clean test -Pcheck_cpp_files -Dtest.data.root="/tmp/custom_path"

* *

Calling the binary test jar: see src/test/resources/testng.xml

* diff --git a/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java b/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java index 97fdc1601..ef75dbc20 100644 --- a/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java +++ b/src/test/java/org/apache/datasketches/sampling/EbppsItemsSketchTest.java @@ -324,4 +324,65 @@ public void deserializeTooShort() { final MemorySegment shortSeg = seg.asSlice(0, seg.byteSize() - 1); EbppsItemsSketch.heapify(shortSeg, new ArrayOfLongsSerDe()); } + + // A merge must keep the larger maximum weight of the two sketches. Otherwise a later update + // uses the smaller one and the sample stops being proportional to weight. + // C depends only on the sequence of rho values, so these checks are deterministic. + // See apache/datasketches-cpp#581. + @Test + public void mergeKeepsMaxWeight() { + final int k = 10; + final int n = 1000; + final double heavyWt = 500.0; + + // reference: the same items in a single sketch + final EbppsItemsSketch ref = new EbppsItemsSketch<>(k); + for (long i = 0; i < n; ++i) { ref.update(i, 1.0); } + ref.update(-1L, heavyWt); + ref.update(-2L, 1.0); + assertEquals(ref.getC(), (n + 1 + heavyWt) / heavyWt, EPS); + + // the heavy item merged into the larger sketch + final EbppsItemsSketch big = buildUnitWeightSketch(k, n); + big.merge(buildHeavySketch(k, heavyWt)); + checkAfterOneUpdate(big, ref); + + // the larger sketch merged into the heavy one: merge() takes the swap path + final EbppsItemsSketch heavy = buildHeavySketch(k, heavyWt); + heavy.merge(buildUnitWeightSketch(k, n)); + checkAfterOneUpdate(heavy, ref); + + // the maximum weight must also survive serialization + final EbppsItemsSketch big2 = buildUnitWeightSketch(k, n); + big2.merge(buildHeavySketch(k, heavyWt)); + checkAfterOneUpdate(heapifyCopy(big2), ref); + + final EbppsItemsSketch heavy2 = buildHeavySketch(k, heavyWt); + heavy2.merge(buildUnitWeightSketch(k, n)); + checkAfterOneUpdate(heapifyCopy(heavy2), ref); + } + + private static EbppsItemsSketch buildUnitWeightSketch(final int k, final int n) { + final EbppsItemsSketch sk = new EbppsItemsSketch<>(k); + for (long i = 0; i < n; ++i) { sk.update(i, 1.0); } + return sk; + } + + private static EbppsItemsSketch buildHeavySketch(final int k, final double heavyWt) { + final EbppsItemsSketch sk = new EbppsItemsSketch<>(k); + sk.update(-1L, heavyWt); + return sk; + } + + private static EbppsItemsSketch heapifyCopy(final EbppsItemsSketch sk) { + final byte[] bytes = sk.toByteArray(new ArrayOfLongsSerDe()); + return EbppsItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + } + + private static void checkAfterOneUpdate(final EbppsItemsSketch sk, final EbppsItemsSketch ref) { + sk.update(-2L, 1.0); + assertEquals(sk.getN(), ref.getN()); + assertEquals(sk.getCumulativeWeight(), ref.getCumulativeWeight(), EPS); + assertEquals(sk.getC(), ref.getC(), EPS); + } }