diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
index 6638007b6..542c43064 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
index 552df0ffb..d6fced949 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
index d4c89322b..739add325 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java
@@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}
- @Override
- void incNumLevels() {
- if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
- int numLevels = getMemorySegmentNumLevels(wseg);
- setMemorySegmentNumLevels(wseg, ++numLevels);
- }
@Override
boolean isLevelZeroSorted() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
index 93b4ec007..46a00f3ef 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
@@ -183,14 +183,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
- ? myCurDoubleItemsArr
- : new double[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -204,6 +197,16 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
+ ? myCurDoubleItemsArr
+ : new double[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);
+
//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
index 9adaca31f..d187dc1d6 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
@@ -334,7 +334,6 @@ public void update(final double item) {
//Also Called from KllDoublesHelper::merge
static void updateDouble(final KllDoublesSketch dblSk, final double item) {
- dblSk.updateMinMax(item);
int freeSpace = dblSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
@@ -342,6 +341,8 @@ static void updateDouble(final KllDoublesSketch dblSk, final double item) {
freeSpace = dblSk.levelsArr[0];
assert (freeSpace > 0);
}
+ //min and max change only after the compression, which may fail, has succeeded
+ dblSk.updateMinMax(item);
dblSk.incN(1);
dblSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
index 7038e1b2f..78633db56 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
@@ -183,14 +183,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger
- // THE NEW ITEMS ARRAY
- myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
- ? myCurFloatItemsArr
- : new float[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;
-
- //shift the new items array create space at bottom
- System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];
//calculate the new levels array length
@@ -204,6 +197,16 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}
+ // THE NEW ITEMS ARRAY
+ //the levels array is allocated first, because this may be the current items array,
+ //and once it is overwritten nothing else may fail
+ myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
+ ? myCurFloatItemsArr
+ : new float[targetItemCount];
+
+ //shift the new items array create space at bottom
+ System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);
+
//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
index 90e3da5fe..2519d31a2 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
@@ -334,7 +334,6 @@ public void update(final float item) {
//Also Called from KllFloatsHelper::merge
static void updateFloat(final KllFloatsSketch fltSk, final float item) {
- fltSk.updateMinMax(item);
int freeSpace = fltSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
@@ -342,6 +341,8 @@ static void updateFloat(final KllFloatsSketch fltSk, final float item) {
freeSpace = fltSk.levelsArr[0];
assert (freeSpace > 0);
}
+ //min and max change only after the compression, which may fail, has succeeded
+ fltSk.updateMinMax(item);
fltSk.incN(1);
fltSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
index 59e391b47..2e1c39d08 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
index 9704f9887..c987aae2e 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
index 864399808..3f6c6c040 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java
@@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }
- @Override
- void incNumLevels() {
- //the heap sketch computes num levels from the array itself, so this is not used on-heap
- }
@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java
index 36492ad87..d1083362b 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java
@@ -661,7 +661,6 @@ else if (sketchType == KLL_LONGS_SKETCH) {
myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2);
assert myNewLevelsArr.length == (myCurLevelsArr.length + 1);
myNewNumLevels = myCurNumLevels + 1;
- sketch.incNumLevels(); //increment for off-heap
} else {
myNewLevelsArr = myCurLevelsArr;
myNewNumLevels = myCurNumLevels;
diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
index a066bf536..2ea651e13 100644
--- a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
@@ -63,11 +63,17 @@ static
Exceptions during update. If an update of a single item throws because storage cannot be + * allocated, because the {@code MemorySegmentRequest} of a sketch backed by a MemorySegment cannot provide a + * larger segment, or because the comparator of an items sketch throws, the item is not added and the sketch + * is unchanged. Weighted and vector updates, and merges, may have applied part of their input when they throw.
+ * * @see KLL Sketch * @see QuantilesAPI * @@ -451,8 +456,6 @@ final int getNumLevels() { abstract void incN(int increment); - abstract void incNumLevels(); - final boolean isCompactSingleItem() { return hasMemorySegment() && (sketchStructure == COMPACT_SINGLE) && (getN() == 1); } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java index 8db7857f5..8110729ed 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java @@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() { final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k); final KllDoublesSketch sk2 = KllDoublesSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setDoubleItemsArray(dblArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setDoubleItemsArrayAt(idx, dblV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java index 39798ae93..2bd37725a 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java @@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() { final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); final KllFloatsSketch sk2 = KllFloatsSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setFloatItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setFloatItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java index 277c3a403..96fcbc523 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java @@ -609,7 +609,7 @@ public void checkReadOnlyExceptions() { final KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); final KllLongsSketch sk2 = KllLongsSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLongItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLongItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java new file mode 100644 index 000000000..cd199081c --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.MemorySegmentRequest; +import org.testng.annotations.Test; + +/** + * Checks that an update whose compaction fails leaves a KLL sketch consistent: + * the item is not added, and min and max are unchanged. + */ +public class KllUpdateFailureTest { + private static final int K = 20; + private static final long SEED = 12345; + + /** A MemorySegmentRequest that refuses every request. */ + private static final MemorySegmentRequest REFUSE = new MemorySegmentRequest() { + @Override + public MemorySegment request(final long newByteSize) { + throw new IllegalStateException("no more space"); + } + }; + + //A direct sketch whose segment cannot grow: the update that needs a new level fails, + //and the segment must still hold the sketch as it was before that update. + @Test + public void directDoublesRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllDoublesSketch.newHeapInstance(K), true)); + final KllDoublesSketch sk = KllDoublesSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(1000.0); //a new maximum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMaxItem(), K - 1.0); + final KllDoublesSketch copy = KllDoublesSketch.heapify(seg); + assertEquals(copy.getN(), K); + assertEquals(copy.getNumLevels(), 1); + assertEquals(copy.getMaxItem(), K - 1.0); + } + + @Test + public void directFloatsRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllFloatsSketch.newHeapInstance(K), true)); + final KllFloatsSketch sk = KllFloatsSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(-1000.0f); //a new minimum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMinItem(), 0.0f); + assertEquals(KllFloatsSketch.heapify(seg).getNumLevels(), 1); + } + + @Test + public void directLongsRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllLongsSketch.newHeapInstance(K), true)); + final KllLongsSketch sk = KllLongsSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(1000L); //a new maximum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMaxItem(), K - 1L); + assertEquals(KllLongsSketch.heapify(seg).getNumLevels(), 1); + } + + /** A comparator that throws once a given number of further comparisons have been made. */ + private static final class FailingComparator implements Comparator