From b301408588565236d7a41809741d7f09d18ac26d Mon Sep 17 00:00:00 2001 From: hutiefang76 <137664623+hutiefang76@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:24:04 +0800 Subject: [PATCH 1/2] Use a bounded-long draw for reservoir updates Reuse the accepted bounded-long value as the replacement index. Cover long item-count boundaries and preserved serialization/capacity behavior. Generated-by: DeepSeek (deepseek-v4-pro) Generated-by: OpenAI Codex --- .../sampling/ReservoirItemsSketch.java | 8 +- .../sampling/ReservoirItemsSketchTest.java | 98 +++++++++++++++++++ 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java index e8ad5f23b..de047c87e 100644 --- a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java +++ b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java @@ -320,10 +320,10 @@ public void update(final T item) { ++itemsSeen_; } else { // code for steady state where we sample randomly ++itemsSeen_; - // prob(keep_item) < k / n = reservoirSize_ / itemsSeen_ - // so multiply to get: keep if rand * itemsSeen_ < reservoirSize_ - if ((SamplingUtil.rand().nextDouble() * itemsSeen_) < reservoirSize_) { - final int newSlot = SamplingUtil.rand().nextInt(reservoirSize_); + // Use the same draw for acceptance and the replacement slot. + final long randomSlot = SamplingUtil.rand().nextLong(itemsSeen_); + if (randomSlot < reservoirSize_) { + final int newSlot = (int) randomSlot; data_.set(newSlot, item); } } diff --git a/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java b/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java index 4dd987b32..dac3abd75 100644 --- a/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java +++ b/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java @@ -36,6 +36,7 @@ import java.lang.foreign.MemorySegment; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.HashSet; import org.apache.datasketches.common.ArrayOfLongsSerDe; import org.apache.datasketches.common.ArrayOfNumbersSerDe; @@ -45,6 +46,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesException; import org.apache.datasketches.common.SketchesStateException; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class ReservoirItemsSketchTest { @@ -595,6 +597,102 @@ public void checkEstimateSubsetSum() { assertTrue(passLB >= 2 && passUB >= 2); //2 out of 3 must pass for LB and UB } + @DataProvider(name = "largeNProvider") + public Object[][] largeNProvider() { + final long[] largeNs = { + (1L << 31) - 1, (1L << 31) + 3, (1L << 32) - 1, (1L << 32) + 3, (1L << 48) - 2 + }; + final int[] kValues = {2, 17}; + final Object[][] params = new Object[kValues.length * (largeNs.length + 1)][2]; + int idx = 0; + for (final int k : kValues) { + params[idx++] = new Object[] {k, (long) k}; + for (final long n : largeNs) { + params[idx++] = new Object[] {k, n}; + } + } + return params; + } + + @Test(dataProvider = "largeNProvider") + public void checkLargeNItemSeenInvariants(final int k, final long n) { + final long newItem = Long.MAX_VALUE; + final ArrayList data = new ArrayList<>(k); + for (long i = 0; i < k; ++i) { + data.add(i); + } + final ReservoirItemsSketch sketch = + ReservoirItemsSketch.newInstance(data, n, ResizeFactor.X8, k); + assertEquals(sketch.getN(), n); + assertEquals(sketch.getNumSamples(), k); + sketch.update(newItem); + assertEquals(sketch.getN(), n + 1); + assertEquals(sketch.getNumSamples(), k); + + final Long[] samples = sketch.getSamples(); + assertEquals(samples.length, k); + final HashSet seen = new HashSet<>(); + for (final Long sample : samples) { + assertTrue(sample == newItem || (sample >= 0 && sample < k)); + assertTrue(seen.add(sample)); // no duplicates + } + + final byte[] bytes = sketch.toByteArray(new ArrayOfLongsSerDe()); + final MemorySegment seg = MemorySegment.ofArray(bytes); + final ReservoirItemsSketch rebuilt = ReservoirItemsSketch.heapify(seg, new ArrayOfLongsSerDe()); + validateReservoirEquality(sketch, rebuilt); + assertEquals( + rebuilt.toByteArray(new ArrayOfLongsSerDe()), sketch.toByteArray(new ArrayOfLongsSerDe())); + } + + @Test + public void checkNearMaxItemSeenCapacity() { + final int k = 2; + final long max = 0xFFFFFFFFFFFFL; // MAX_ITEMS_SEEN + final ArrayList data = new ArrayList<>(k); + data.add(0L); + data.add(1L); + final ReservoirItemsSketch sketch = + ReservoirItemsSketch.newInstance(data, max - 1, ResizeFactor.X8, k); + + final byte[] beforeNull = sketch.toByteArray(new ArrayOfLongsSerDe()); + sketch.update(null); + assertEquals(sketch.getN(), max - 1); + assertEquals(sketch.toByteArray(new ArrayOfLongsSerDe()), beforeNull); + + sketch.update(2L); + assertEquals(sketch.getN(), max); + assertEquals(sketch.getNumSamples(), k); + final byte[] afterSuccessBytes = sketch.toByteArray(new ArrayOfLongsSerDe()); + final MemorySegment seg = MemorySegment.ofArray(afterSuccessBytes); + final ReservoirItemsSketch rebuilt = ReservoirItemsSketch.heapify(seg, new ArrayOfLongsSerDe()); + assertEquals(rebuilt.getN(), max); + validateReservoirEquality(sketch, rebuilt); + assertEquals( + rebuilt.toByteArray(new ArrayOfLongsSerDe()), sketch.toByteArray(new ArrayOfLongsSerDe())); + + try { + sketch.update(3L); + fail("Expected SketchesStateException"); + } catch (final SketchesStateException e) { + // expected + } + assertEquals(sketch.getN(), max); + assertEquals(sketch.getNumSamples(), k); + final byte[] afterFailedNonnull = sketch.toByteArray(new ArrayOfLongsSerDe()); + assertEquals(afterFailedNonnull, afterSuccessBytes); + + try { + sketch.update(null); + fail("Expected SketchesStateException"); + } catch (final SketchesStateException e) { + // expected + } + assertEquals(sketch.getN(), max); + final byte[] afterFailedNull = sketch.toByteArray(new ArrayOfLongsSerDe()); + assertEquals(afterFailedNull, afterSuccessBytes); + } + private static MemorySegment getBasicSerializedLongsRIS() { final int k = 10; final int n = 20; From cd5c9fb0f9dbd24ed3c40e165005444d014ef2f3 Mon Sep 17 00:00:00 2001 From: hutiefang76 <137664623+hutiefang76@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:13:30 +0800 Subject: [PATCH 2/2] Use one bounded draw in the long reservoir sketch Match the item reservoir update and clarify the null-at-capacity test. Generated-by: DeepSeek (deepseek-v4-pro) Generated-by: OpenAI Codex --- .../datasketches/sampling/ReservoirLongsSketch.java | 8 ++++---- .../datasketches/sampling/ReservoirItemsSketchTest.java | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java index a73210f56..988575bb7 100644 --- a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java +++ b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java @@ -320,10 +320,10 @@ public void update(final long item) { ++itemsSeen_; } else { // code for steady state where we sample randomly ++itemsSeen_; - // prob(keep_item) < k / n = reservoirSize_ / itemsSeen_ - // so multiply to get: keep if rand * itemsSeen_ < reservoirSize_ - if ((SamplingUtil.rand().nextDouble() * itemsSeen_) < reservoirSize_) { - final int newSlot = SamplingUtil.rand().nextInt(reservoirSize_); + // Use the same draw for acceptance and the replacement slot. + final long randomSlot = SamplingUtil.rand().nextLong(itemsSeen_); + if (randomSlot < reservoirSize_) { + final int newSlot = (int) randomSlot; data_[newSlot] = item; } } diff --git a/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java b/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java index dac3abd75..0e625398b 100644 --- a/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java +++ b/src/test/java/org/apache/datasketches/sampling/ReservoirItemsSketchTest.java @@ -682,6 +682,7 @@ public void checkNearMaxItemSeenCapacity() { final byte[] afterFailedNonnull = sketch.toByteArray(new ArrayOfLongsSerDe()); assertEquals(afterFailedNonnull, afterSuccessBytes); + // Pins current behavior: the capacity check runs before the null check. try { sketch.update(null); fail("Expected SketchesStateException");