Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ private void internalMerge(final EbppsItemsSketch<T> other) {
// avoid numeric issues by setting cumulative weight to the
// pre-computed value
cumulativeWt_ = finalCumWt;
wtMax_ = newWtMax;
n_ = newN;
}

Expand Down
4 changes: 2 additions & 2 deletions src/test/java/org/apache/datasketches/common/UtilityIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public static byte[] getTestResourceBytes(final String fileName) {

/**
* Puts all the bytes of the given byte array to a file with the given fileName.
* This assumes that the base directory path is {@link #javaPath javaPath}.
* This assumes that the base directory path is {@link #getJavaPath()}.
* @param fileName the name of the target file
* @param bytes the given byte array
*/
Expand Down Expand Up @@ -229,7 +229,7 @@ static void putBytesToFile(final Path basePath, final String fileName, final byt
* Resolves the target directory for test data reading/writing.
* The default is the project root, but it can be overridden like this:
*
* <p><i>mvn clean test -Pcheck-cpp-files -Dtest.data.root="/tmp/custom_path</i></p>
* <p><i>mvn clean test -Pcheck_cpp_files -Dtest.data.root="/tmp/custom_path"</i></p>
*
* <p>Calling the binary test jar: see src/test/resources/testng.xml</p>
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,4 +324,65 @@ public void deserializeTooShort() {
final MemorySegment shortSeg = seg.asSlice(0, seg.byteSize() - 1);
EbppsItemsSketch.heapify(shortSeg, new ArrayOfLongsSerDe());
}

// A merge must keep the larger maximum weight of the two sketches. Otherwise a later update
// uses the smaller one and the sample stops being proportional to weight.
// C depends only on the sequence of rho values, so these checks are deterministic.
// See apache/datasketches-cpp#581.
@Test
public void mergeKeepsMaxWeight() {
final int k = 10;
final int n = 1000;
final double heavyWt = 500.0;

// reference: the same items in a single sketch
final EbppsItemsSketch<Long> ref = new EbppsItemsSketch<>(k);
for (long i = 0; i < n; ++i) { ref.update(i, 1.0); }
ref.update(-1L, heavyWt);
ref.update(-2L, 1.0);
assertEquals(ref.getC(), (n + 1 + heavyWt) / heavyWt, EPS);

// the heavy item merged into the larger sketch
final EbppsItemsSketch<Long> big = buildUnitWeightSketch(k, n);
big.merge(buildHeavySketch(k, heavyWt));
checkAfterOneUpdate(big, ref);

// the larger sketch merged into the heavy one: merge() takes the swap path
final EbppsItemsSketch<Long> heavy = buildHeavySketch(k, heavyWt);
heavy.merge(buildUnitWeightSketch(k, n));
checkAfterOneUpdate(heavy, ref);

// the maximum weight must also survive serialization
final EbppsItemsSketch<Long> big2 = buildUnitWeightSketch(k, n);
big2.merge(buildHeavySketch(k, heavyWt));
checkAfterOneUpdate(heapifyCopy(big2), ref);

final EbppsItemsSketch<Long> heavy2 = buildHeavySketch(k, heavyWt);
heavy2.merge(buildUnitWeightSketch(k, n));
checkAfterOneUpdate(heapifyCopy(heavy2), ref);
}

private static EbppsItemsSketch<Long> buildUnitWeightSketch(final int k, final int n) {
final EbppsItemsSketch<Long> sk = new EbppsItemsSketch<>(k);
for (long i = 0; i < n; ++i) { sk.update(i, 1.0); }
return sk;
}

private static EbppsItemsSketch<Long> buildHeavySketch(final int k, final double heavyWt) {
final EbppsItemsSketch<Long> sk = new EbppsItemsSketch<>(k);
sk.update(-1L, heavyWt);
return sk;
}

private static EbppsItemsSketch<Long> heapifyCopy(final EbppsItemsSketch<Long> sk) {
final byte[] bytes = sk.toByteArray(new ArrayOfLongsSerDe());
return EbppsItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe());
}

private static void checkAfterOneUpdate(final EbppsItemsSketch<Long> sk, final EbppsItemsSketch<Long> ref) {
sk.update(-2L, 1.0);
assertEquals(sk.getN(), ref.getN());
assertEquals(sk.getCumulativeWeight(), ref.getCumulativeWeight(), EPS);
assertEquals(sk.getC(), ref.getC(), EPS);
}
}
Loading