From 09384e5a0de53469e871f55553850f54457955df Mon Sep 17 00:00:00 2001
From: Lee Rhodes
Date: Wed, 18 Nov 2026 18:35:41 -0800
Subject: [PATCH] KLL: keep the sketch consistent when a single-item update
fails
- A direct doubles, floats or longs sketch wrote numLevels + 1 into its
segment before asking for a larger one. If that request failed, the
segment was left with one level too many. The early write
(incNumLevels) is removed; setNumLevels at the end already writes the
right value, so incNumLevels is gone.
- A single-item update set min and max before compaction, which can
fail. They are now set after it.
- An items sketch whose comparator throws during compaction now removes
the top level it had just added, so the sketch is unchanged.
- Merge allocates the new levels array before it may overwrite the
current items array.
Results and the serialized form are unchanged.
Co-Authored-By: Claude Opus 5.5
---
.../kll/KllDirectDoublesSketch.java | 6 -
.../kll/KllDirectFloatsSketch.java | 6 -
.../kll/KllDirectLongsSketch.java | 6 -
.../datasketches/kll/KllDoublesHelper.java | 17 +-
.../datasketches/kll/KllDoublesSketch.java | 3 +-
.../datasketches/kll/KllFloatsHelper.java | 17 +-
.../datasketches/kll/KllFloatsSketch.java | 3 +-
.../kll/KllHeapDoublesSketch.java | 4 -
.../datasketches/kll/KllHeapFloatsSketch.java | 4 -
.../datasketches/kll/KllHeapLongsSketch.java | 4 -
.../apache/datasketches/kll/KllHelper.java | 1 -
.../datasketches/kll/KllItemsHelper.java | 72 +++++---
.../datasketches/kll/KllItemsSketch.java | 14 --
.../datasketches/kll/KllLongsHelper.java | 17 +-
.../datasketches/kll/KllLongsSketch.java | 3 +-
.../apache/datasketches/kll/KllSketch.java | 7 +-
.../kll/KllDirectDoublesSketchTest.java | 2 +-
.../kll/KllDirectFloatsSketchTest.java | 2 +-
.../kll/KllDirectLongsSketchTest.java | 2 +-
.../kll/KllUpdateFailureTest.java | 165 ++++++++++++++++++
20 files changed, 259 insertions(+), 96 deletions(-)
create mode 100644 src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java
diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
index 6638007b6..542c43064 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
index 552df0ffb..d6fced949 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
index d4c89322b..739add325 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
index 93b4ec007..46a00f3ef 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
@@ -183,14 +183,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
- ? myCurDoubleItemsArr
- : new double[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -204,6 +197,16 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
+ ? myCurDoubleItemsArr
+ : new double[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);
+
//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
index 9adaca31f..d187dc1d6 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
@@ -334,7 +334,6 @@ public void update(final double item) {
//Also Called from KllDoublesHelper::merge
static void updateDouble(final KllDoublesSketch dblSk, final double item) {
- dblSk.updateMinMax(item);
int freeSpace = dblSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
@@ -342,6 +341,8 @@ static void updateDouble(final KllDoublesSketch dblSk, final double item) {
freeSpace = dblSk.levelsArr[0];
assert (freeSpace > 0);
}
+ //min and max change only after the compression, which may fail, has succeeded
+ dblSk.updateMinMax(item);
dblSk.incN(1);
dblSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
index 7038e1b2f..78633db56 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
@@ -183,14 +183,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
- ? myCurFloatItemsArr
- : new float[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -204,6 +197,16 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
+ ? myCurFloatItemsArr
+ : new float[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);
+
//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
index 90e3da5fe..2519d31a2 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
@@ -334,7 +334,6 @@ public void update(final float item) {
//Also Called from KllFloatsHelper::merge
static void updateFloat(final KllFloatsSketch fltSk, final float item) {
- fltSk.updateMinMax(item);
int freeSpace = fltSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
@@ -342,6 +341,8 @@ static void updateFloat(final KllFloatsSketch fltSk, final float item) {
freeSpace = fltSk.levelsArr[0];
assert (freeSpace > 0);
}
+ //min and max change only after the compression, which may fail, has succeeded
+ fltSk.updateMinMax(item);
fltSk.incN(1);
fltSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
index 59e391b47..2e1c39d08 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
index 9704f9887..c987aae2e 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
index 864399808..3f6c6c040 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java
index 36492ad87..d1083362b 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java
@@ -661,7 +661,6 @@ else if (sketchType == KLL_LONGS_SKETCH) {
myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2);
assert myNewLevelsArr.length == (myCurLevelsArr.length + 1);
myNewNumLevels = myCurNumLevels + 1;
- sketch.incNumLevels(); //increment for off-heap
} else {
myNewLevelsArr = myCurLevelsArr;
myNewNumLevels = myCurNumLevels;
diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
index a066bf536..2ea651e13 100644
--- a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
@@ -63,11 +63,17 @@ static T[] createItemsArray(final Class clazz, final T item, final long w
private static void compressWhileUpdatingSketch(final KllItemsSketch itmSk) {
final int level =
findLevelToCompact(itmSk.getK(), itmSk.getM(), itmSk.getNumLevels(), itmSk.levelsArr);
+ //The comparator may throw while sorting or merging below. Nothing in the sketch changes before
+ //that work is done, except for a new top level, which is then removed again.
+ int[] savedLevelsArr = null;
+ Object[] savedItemsArr = null;
if (level == (itmSk.getNumLevels() - 1)) {
//The level to compact is the top level, thus we need to add a level.
//Be aware that this operation grows the items array,
//shifts the items data and the level boundaries of the data,
//and grows the levels array and increments numLevels_.
+ savedLevelsArr = itmSk.levelsArr;
+ savedItemsArr = itmSk.getTotalItemsArray(); //a copy; growth happens only O(log n) times
KllHelper.addEmptyTopLevelToCompletelyFullSketch(itmSk);
}
//after this point, the levelsArray will not be expanded, only modified.
@@ -83,18 +89,27 @@ private static void compressWhileUpdatingSketch(final KllItemsSketch itmS
final int halfAdjPop = adjPop / 2;
//the following is specific to generic Items
- final Object[] myItemsArr = itmSk.getTotalItemsArray();
- if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it
- Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator);
- }
- if (popAbove == 0) {
- KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
- } else {
- KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
- KllItemsHelper.mergeSortedItemsArrays(
- myItemsArr, adjBeg, halfAdjPop,
- myItemsArr, rawEnd, popAbove,
- myItemsArr, adjBeg + halfAdjPop, itmSk.comparator);
+ final Object[] myItemsArr;
+ try {
+ myItemsArr = itmSk.getTotalItemsArray(); //a copy, set into the sketch at the end
+ if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it
+ Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator);
+ }
+ if (popAbove == 0) {
+ KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
+ } else {
+ KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
+ KllItemsHelper.mergeSortedItemsArrays(
+ myItemsArr, adjBeg, halfAdjPop,
+ myItemsArr, rawEnd, popAbove,
+ myItemsArr, adjBeg + halfAdjPop, itmSk.comparator);
+ }
+ } catch (final RuntimeException | Error e) {
+ if (savedLevelsArr != null) { //remove the new top level
+ itmSk.setLevelsArray(savedLevelsArr);
+ itmSk.setItemsArray(savedItemsArr);
+ }
+ throw e;
}
int newIndex = myLevelsArr[level + 1] - halfAdjPop; // adjust boundaries of the level above
@@ -189,14 +204,7 @@ static void mergeItemImpl(final KllItemsSketch mySketch,
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewItemsArr = (targetItemCount == myCurItemsArr.length)
- ? myCurItemsArr
- : new Object[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -210,6 +218,16 @@ static void mergeItemImpl(final KllItemsSketch mySketch,
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewItemsArr = (targetItemCount == myCurItemsArr.length)
+ ? myCurItemsArr
+ : new Object[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount);
+
//MEMORY SPACE MANAGEMENT
//not used
//extra spaces to make comparison with other helpers easier
@@ -317,7 +335,16 @@ private static void randomlyHalveUpItems(final Object[] buf, final int start, fi
//Called from KllItemsSketch::update and this
static void updateItem(final KllItemsSketch itmSk, final Object item) {
- itmSk.updateMinMax((T)item);
+ //compare first, but change min and max only after the compression, which may fail, has succeeded
+ final T newMin;
+ final T newMax;
+ if (itmSk.isEmpty()) {
+ newMin = (T) item;
+ newMax = (T) item;
+ } else {
+ newMin = (T) Util.minT(itmSk.getMinItem(), item, itmSk.comparator);
+ newMax = (T) Util.maxT(itmSk.getMaxItem(), item, itmSk.comparator);
+ }
int freeSpace = itmSk.levelsArr[0];
assert freeSpace >= 0;
if (freeSpace == 0) {
@@ -325,6 +352,8 @@ static void updateItem(final KllItemsSketch itmSk, final Object item) {
freeSpace = itmSk.levelsArr[0];
assert (freeSpace > 0);
}
+ itmSk.setMinItem(newMin);
+ itmSk.setMaxItem(newMax);
itmSk.incN(1);
itmSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
@@ -336,8 +365,7 @@ static void updateItem(final KllItemsSketch itmSk, final Object item) {
static void updateItem(final KllItemsSketch itmSk, final T item, final long weight) {
if (weight < itmSk.levelsArr[0]) {
for (int i = 0; i < (int)weight; i++) { updateItem(itmSk, item); }
- } else {
- itmSk.updateMinMax(item);
+ } else { //the merge updates min and max
final KllHeapItemsSketch tmpSk =
new KllHeapItemsSketch<>(itmSk.getK(), DEFAULT_M, item, weight, itmSk.comparator, itmSk.serDe);
itmSk.merge(tmpSk);
diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
index 5c147b2c7..a39ad0823 100644
--- a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
@@ -378,10 +378,6 @@ int getTotalItemsNumBytes() {
throw new SketchesArgumentException(UNSUPPORTED_MSG);
}
- @Override
- void incNumLevels() {
- //this is not used and must be a no-op.
- }
abstract void setItemsArray(Object[] ItemsArr);
@@ -401,16 +397,6 @@ void setMemorySegment(final MemorySegment wseg) {
throw new SketchesArgumentException(UNSUPPORTED_MSG + "Sketch not writable.");
}
- void updateMinMax(final T item) {
- if (isEmpty()) {
- setMinItem(item);
- setMaxItem(item);
- } else {
- setMinItem(Util.minT(getMinItem(), item, comparator));
- setMaxItem(Util.maxT(getMaxItem(), item, comparator));
- }
- }
-
private final ItemsSketchSortedView refreshSortedView() {
if (itemsSV == null) {
final CreateSortedView csv = new CreateSortedView();
diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
index 94638b047..3be9956a7 100644
--- a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
@@ -183,14 +183,7 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length)
- ? myCurLongItemsArray
- : new long[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -204,6 +197,16 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length)
+ ? myCurLongItemsArray
+ : new long[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount);
+
//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length);
diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
index 22e6925a1..2c4312517 100644
--- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java
@@ -334,7 +334,6 @@ public void update(final long item) {
//Also Called from KllLongsHelper::merge
static void updateLong(final KllLongsSketch lngSk, final long item) {
- lngSk.updateMinMax(item);
int freeSpace = lngSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
@@ -342,6 +341,8 @@ static void updateLong(final KllLongsSketch lngSk, final long item) {
freeSpace = lngSk.levelsArr[0];
assert (freeSpace > 0);
}
+ //min and max change only after the compression, which may fail, has succeeded
+ lngSk.updateMinMax(item);
lngSk.incN(1);
lngSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
diff --git a/src/main/java/org/apache/datasketches/kll/KllSketch.java b/src/main/java/org/apache/datasketches/kll/KllSketch.java
index a91ee50e8..0ef42feff 100644
--- a/src/main/java/org/apache/datasketches/kll/KllSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllSketch.java
@@ -89,6 +89,11 @@
* the storage up front instead. A sketch backed by a MemorySegment is not affected.
* Results and the serialized form are the same either way.
*
+ * Exceptions during update. If an update of a single item throws because storage cannot be
+ * allocated, because the {@code MemorySegmentRequest} of a sketch backed by a MemorySegment cannot provide a
+ * larger segment, or because the comparator of an items sketch throws, the item is not added and the sketch
+ * is unchanged. Weighted and vector updates, and merges, may have applied part of their input when they throw.
+ *
* @see KLL Sketch
* @see QuantilesAPI
*
@@ -451,8 +456,6 @@ final int getNumLevels() {
abstract void incN(int increment);
- abstract void incNumLevels();
-
final boolean isCompactSingleItem() {
return hasMemorySegment() && (sketchStructure == COMPACT_SINGLE) && (getN() == 1);
}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
index 8db7857f5..8110729ed 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
@@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() {
final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k);
final KllDoublesSketch sk2 = KllDoublesSketch.wrap(MemorySegment.ofArray(sk.toByteArray()));
try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { }
- try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { }
+ try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setDoubleItemsArray(dblArr); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setDoubleItemsArrayAt(idx, dblV); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { }
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java
index 39798ae93..2bd37725a 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java
@@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() {
final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k);
final KllFloatsSketch sk2 = KllFloatsSketch.wrap(MemorySegment.ofArray(sk.toByteArray()));
try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { }
- try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { }
+ try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setFloatItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setFloatItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { }
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java
index 277c3a403..96fcbc523 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java
@@ -609,7 +609,7 @@ public void checkReadOnlyExceptions() {
final KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
final KllLongsSketch sk2 = KllLongsSketch.wrap(MemorySegment.ofArray(sk.toByteArray()));
try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { }
- try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { }
+ try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setLongItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setLongItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { }
try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { }
diff --git a/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java
new file mode 100644
index 000000000..cd199081c
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+import java.lang.foreign.MemorySegment;
+import java.util.Comparator;
+
+import org.apache.datasketches.common.ArrayOfStringsSerDe;
+import org.apache.datasketches.common.MemorySegmentRequest;
+import org.testng.annotations.Test;
+
+/**
+ * Checks that an update whose compaction fails leaves a KLL sketch consistent:
+ * the item is not added, and min and max are unchanged.
+ */
+public class KllUpdateFailureTest {
+ private static final int K = 20;
+ private static final long SEED = 12345;
+
+ /** A MemorySegmentRequest that refuses every request. */
+ private static final MemorySegmentRequest REFUSE = new MemorySegmentRequest() {
+ @Override
+ public MemorySegment request(final long newByteSize) {
+ throw new IllegalStateException("no more space");
+ }
+ };
+
+ //A direct sketch whose segment cannot grow: the update that needs a new level fails,
+ //and the segment must still hold the sketch as it was before that update.
+ @Test
+ public void directDoublesRefusedGrowthLeavesSegmentUnchanged() {
+ final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllDoublesSketch.newHeapInstance(K), true));
+ final KllDoublesSketch sk = KllDoublesSketch.newDirectInstance(K, seg, REFUSE);
+ for (int i = 0; i < K; i++) { sk.update(i); }
+ final byte[] before = seg.toArray(JAVA_BYTE);
+ try {
+ sk.update(1000.0); //a new maximum
+ fail("the segment request should have been refused");
+ } catch (final IllegalStateException e) {
+ //expected
+ }
+ assertEquals(seg.toArray(JAVA_BYTE), before);
+ assertEquals(sk.getN(), K);
+ assertEquals(sk.getMaxItem(), K - 1.0);
+ final KllDoublesSketch copy = KllDoublesSketch.heapify(seg);
+ assertEquals(copy.getN(), K);
+ assertEquals(copy.getNumLevels(), 1);
+ assertEquals(copy.getMaxItem(), K - 1.0);
+ }
+
+ @Test
+ public void directFloatsRefusedGrowthLeavesSegmentUnchanged() {
+ final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllFloatsSketch.newHeapInstance(K), true));
+ final KllFloatsSketch sk = KllFloatsSketch.newDirectInstance(K, seg, REFUSE);
+ for (int i = 0; i < K; i++) { sk.update(i); }
+ final byte[] before = seg.toArray(JAVA_BYTE);
+ try {
+ sk.update(-1000.0f); //a new minimum
+ fail("the segment request should have been refused");
+ } catch (final IllegalStateException e) {
+ //expected
+ }
+ assertEquals(seg.toArray(JAVA_BYTE), before);
+ assertEquals(sk.getN(), K);
+ assertEquals(sk.getMinItem(), 0.0f);
+ assertEquals(KllFloatsSketch.heapify(seg).getNumLevels(), 1);
+ }
+
+ @Test
+ public void directLongsRefusedGrowthLeavesSegmentUnchanged() {
+ final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllLongsSketch.newHeapInstance(K), true));
+ final KllLongsSketch sk = KllLongsSketch.newDirectInstance(K, seg, REFUSE);
+ for (int i = 0; i < K; i++) { sk.update(i); }
+ final byte[] before = seg.toArray(JAVA_BYTE);
+ try {
+ sk.update(1000L); //a new maximum
+ fail("the segment request should have been refused");
+ } catch (final IllegalStateException e) {
+ //expected
+ }
+ assertEquals(seg.toArray(JAVA_BYTE), before);
+ assertEquals(sk.getN(), K);
+ assertEquals(sk.getMaxItem(), K - 1L);
+ assertEquals(KllLongsSketch.heapify(seg).getNumLevels(), 1);
+ }
+
+ /** A comparator that throws once a given number of further comparisons have been made. */
+ private static final class FailingComparator implements Comparator {
+ private int allowed = -1; //negative: never throws
+
+ void failAfter(final int comparisons) { allowed = comparisons; }
+
+ @Override
+ public int compare(final String a, final String b) {
+ if (allowed == 0) {
+ allowed = -1;
+ throw new IllegalStateException("comparator failure");
+ }
+ if (allowed > 0) { allowed--; }
+ return a.compareTo(b);
+ }
+ }
+
+ private static String item(final int i) { return String.format("%06d", (i * 7919) % 100_003); }
+
+ //The comparator throws while level 0 is sorted for compaction. The failed item is not added,
+ //min and max are unchanged, and after more updates the sketch matches one that never saw that item.
+ @Test
+ public void itemsComparatorFailureLeavesSketchConsistent() {
+ final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
+ final int failAt = K; //the update that finds level 0 full
+ final int total = 50 * K;
+
+ KllSketch.random.setSeed(SEED);
+ final FailingComparator comparator = new FailingComparator();
+ final KllItemsSketch sk = KllItemsSketch.newHeapInstance(K, comparator, serDe);
+ for (int i = 0; i < total; i++) {
+ if (i == failAt) {
+ final String min = sk.getMinItem();
+ final String max = sk.getMaxItem();
+ comparator.failAfter(2); //the min and max checks pass, the sort fails
+ try {
+ sk.update("~"); //sorts after every item, so a new maximum
+ fail("the comparator should have thrown");
+ } catch (final IllegalStateException e) {
+ //expected
+ }
+ assertEquals(sk.getN(), failAt);
+ assertEquals(sk.getMinItem(), min);
+ assertEquals(sk.getMaxItem(), max);
+ }
+ sk.update(item(i));
+ }
+
+ KllSketch.random.setSeed(SEED);
+ final KllItemsSketch ref = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), serDe);
+ for (int i = 0; i < total; i++) { ref.update(item(i)); }
+
+ assertEquals(sk.getN(), total);
+ assertEquals(sk.getMinItem(), ref.getMinItem());
+ assertEquals(sk.getMaxItem(), ref.getMaxItem());
+ assertEquals(sk.toByteArray(), ref.toByteArray());
+ }
+}