diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java index 6638007b6..542c43064 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java @@ -317,12 +317,6 @@ void incN(final int increment) { setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment); } - @Override - void incNumLevels() { - if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } - int numLevels = getMemorySegmentNumLevels(wseg); - setMemorySegmentNumLevels(wseg, ++numLevels); - } @Override boolean isLevelZeroSorted() { diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java index 552df0ffb..d6fced949 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java @@ -317,12 +317,6 @@ void incN(final int increment) { setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment); } - @Override - void incNumLevels() { - if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } - int numLevels = getMemorySegmentNumLevels(wseg); - setMemorySegmentNumLevels(wseg, ++numLevels); - } @Override boolean isLevelZeroSorted() { diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java index d4c89322b..739add325 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java @@ -317,12 +317,6 @@ void incN(final int increment) { setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment); } - @Override - void incNumLevels() { - if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } - int numLevels = getMemorySegmentNumLevels(wseg); - setMemorySegmentNumLevels(wseg, ++numLevels); - } @Override boolean isLevelZeroSorted() { diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java index 93b4ec007..46a00f3ef 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java @@ -183,14 +183,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke myNewNumLevels = result[0]; assert myNewNumLevels <= ub; // ub may be much bigger - // THE NEW ITEMS ARRAY - myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length) - ? myCurDoubleItemsArr - : new double[targetItemCount]; final int freeSpaceAtBottom = targetItemCount - curItemCount; - - //shift the new items array create space at bottom - System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount); final int theShift = freeSpaceAtBottom - outlevels[0]; //calculate the new levels array length @@ -204,6 +197,16 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke myNewLevelsArr[lvl] = outlevels[lvl] + theShift; } + // THE NEW ITEMS ARRAY + //the levels array is allocated first, because this may be the current items array, + //and once it is overwritten nothing else may fail + myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length) + ? myCurDoubleItemsArr + : new double[targetItemCount]; + + //shift the new items array create space at bottom + System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount); + //MemorySegment SPACE MANAGEMENT if (mySketch.getMemorySegment() != null) { final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length); diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java index 9adaca31f..d187dc1d6 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java @@ -334,7 +334,6 @@ public void update(final double item) { //Also Called from KllDoublesHelper::merge static void updateDouble(final KllDoublesSketch dblSk, final double item) { - dblSk.updateMinMax(item); int freeSpace = dblSk.levelsArr[0]; assert (freeSpace >= 0); if (freeSpace == 0) { @@ -342,6 +341,8 @@ static void updateDouble(final KllDoublesSketch dblSk, final double item) { freeSpace = dblSk.levelsArr[0]; assert (freeSpace > 0); } + //min and max change only after the compression, which may fail, has succeeded + dblSk.updateMinMax(item); dblSk.incN(1); dblSk.setLevelZeroSorted(false); final int nextPos = freeSpace - 1; diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java index 7038e1b2f..78633db56 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java @@ -183,14 +183,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch myNewNumLevels = result[0]; assert myNewNumLevels <= ub; // ub may be much bigger - // THE NEW ITEMS ARRAY - myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length) - ? myCurFloatItemsArr - : new float[targetItemCount]; final int freeSpaceAtBottom = targetItemCount - curItemCount; - - //shift the new items array create space at bottom - System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount); final int theShift = freeSpaceAtBottom - outlevels[0]; //calculate the new levels array length @@ -204,6 +197,16 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch myNewLevelsArr[lvl] = outlevels[lvl] + theShift; } + // THE NEW ITEMS ARRAY + //the levels array is allocated first, because this may be the current items array, + //and once it is overwritten nothing else may fail + myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length) + ? myCurFloatItemsArr + : new float[targetItemCount]; + + //shift the new items array create space at bottom + System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount); + //MemorySegment SPACE MANAGEMENT if (mySketch.getMemorySegment() != null) { final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java index 90e3da5fe..2519d31a2 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java @@ -334,7 +334,6 @@ public void update(final float item) { //Also Called from KllFloatsHelper::merge static void updateFloat(final KllFloatsSketch fltSk, final float item) { - fltSk.updateMinMax(item); int freeSpace = fltSk.levelsArr[0]; assert (freeSpace >= 0); if (freeSpace == 0) { @@ -342,6 +341,8 @@ static void updateFloat(final KllFloatsSketch fltSk, final float item) { freeSpace = fltSk.levelsArr[0]; assert (freeSpace > 0); } + //min and max change only after the compression, which may fail, has succeeded + fltSk.updateMinMax(item); fltSk.incN(1); fltSk.setLevelZeroSorted(false); final int nextPos = freeSpace - 1; diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java index 59e391b47..2e1c39d08 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() { @Override void incN(final int increment) { n += increment; } - @Override - void incNumLevels() { - //the heap sketch computes num levels from the array itself, so this is not used on-heap - } @Override boolean isLevelZeroSorted() { return isLevelZeroSorted; } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java index 9704f9887..c987aae2e 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() { @Override void incN(final int increment) { n += increment; } - @Override - void incNumLevels() { - //the heap sketch computes num levels from the array itself, so this is not used on-heap - } @Override boolean isLevelZeroSorted() { return isLevelZeroSorted; } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java index 864399808..3f6c6c040 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() { @Override void incN(final int increment) { n += increment; } - @Override - void incNumLevels() { - //the heap sketch computes num levels from the array itself, so this is not used on-heap - } @Override boolean isLevelZeroSorted() { return isLevelZeroSorted; } diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index 36492ad87..d1083362b 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -661,7 +661,6 @@ else if (sketchType == KLL_LONGS_SKETCH) { myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2); assert myNewLevelsArr.length == (myCurLevelsArr.length + 1); myNewNumLevels = myCurNumLevels + 1; - sketch.incNumLevels(); //increment for off-heap } else { myNewLevelsArr = myCurLevelsArr; myNewNumLevels = myCurNumLevels; diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java index a066bf536..2ea651e13 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java @@ -63,11 +63,17 @@ static T[] createItemsArray(final Class clazz, final T item, final long w private static void compressWhileUpdatingSketch(final KllItemsSketch itmSk) { final int level = findLevelToCompact(itmSk.getK(), itmSk.getM(), itmSk.getNumLevels(), itmSk.levelsArr); + //The comparator may throw while sorting or merging below. Nothing in the sketch changes before + //that work is done, except for a new top level, which is then removed again. + int[] savedLevelsArr = null; + Object[] savedItemsArr = null; if (level == (itmSk.getNumLevels() - 1)) { //The level to compact is the top level, thus we need to add a level. //Be aware that this operation grows the items array, //shifts the items data and the level boundaries of the data, //and grows the levels array and increments numLevels_. + savedLevelsArr = itmSk.levelsArr; + savedItemsArr = itmSk.getTotalItemsArray(); //a copy; growth happens only O(log n) times KllHelper.addEmptyTopLevelToCompletelyFullSketch(itmSk); } //after this point, the levelsArray will not be expanded, only modified. @@ -83,18 +89,27 @@ private static void compressWhileUpdatingSketch(final KllItemsSketch itmS final int halfAdjPop = adjPop / 2; //the following is specific to generic Items - final Object[] myItemsArr = itmSk.getTotalItemsArray(); - if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it - Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator); - } - if (popAbove == 0) { - KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random); - } else { - KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random); - KllItemsHelper.mergeSortedItemsArrays( - myItemsArr, adjBeg, halfAdjPop, - myItemsArr, rawEnd, popAbove, - myItemsArr, adjBeg + halfAdjPop, itmSk.comparator); + final Object[] myItemsArr; + try { + myItemsArr = itmSk.getTotalItemsArray(); //a copy, set into the sketch at the end + if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it + Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator); + } + if (popAbove == 0) { + KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random); + } else { + KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random); + KllItemsHelper.mergeSortedItemsArrays( + myItemsArr, adjBeg, halfAdjPop, + myItemsArr, rawEnd, popAbove, + myItemsArr, adjBeg + halfAdjPop, itmSk.comparator); + } + } catch (final RuntimeException | Error e) { + if (savedLevelsArr != null) { //remove the new top level + itmSk.setLevelsArray(savedLevelsArr); + itmSk.setItemsArray(savedItemsArr); + } + throw e; } int newIndex = myLevelsArr[level + 1] - halfAdjPop; // adjust boundaries of the level above @@ -189,14 +204,7 @@ static void mergeItemImpl(final KllItemsSketch mySketch, myNewNumLevels = result[0]; assert myNewNumLevels <= ub; // ub may be much bigger - // THE NEW ITEMS ARRAY - myNewItemsArr = (targetItemCount == myCurItemsArr.length) - ? myCurItemsArr - : new Object[targetItemCount]; final int freeSpaceAtBottom = targetItemCount - curItemCount; - - //shift the new items array create space at bottom - System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount); final int theShift = freeSpaceAtBottom - outlevels[0]; //calculate the new levels array length @@ -210,6 +218,16 @@ static void mergeItemImpl(final KllItemsSketch mySketch, myNewLevelsArr[lvl] = outlevels[lvl] + theShift; } + // THE NEW ITEMS ARRAY + //the levels array is allocated first, because this may be the current items array, + //and once it is overwritten nothing else may fail + myNewItemsArr = (targetItemCount == myCurItemsArr.length) + ? myCurItemsArr + : new Object[targetItemCount]; + + //shift the new items array create space at bottom + System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount); + //MEMORY SPACE MANAGEMENT //not used //extra spaces to make comparison with other helpers easier @@ -317,7 +335,16 @@ private static void randomlyHalveUpItems(final Object[] buf, final int start, fi //Called from KllItemsSketch::update and this static void updateItem(final KllItemsSketch itmSk, final Object item) { - itmSk.updateMinMax((T)item); + //compare first, but change min and max only after the compression, which may fail, has succeeded + final T newMin; + final T newMax; + if (itmSk.isEmpty()) { + newMin = (T) item; + newMax = (T) item; + } else { + newMin = (T) Util.minT(itmSk.getMinItem(), item, itmSk.comparator); + newMax = (T) Util.maxT(itmSk.getMaxItem(), item, itmSk.comparator); + } int freeSpace = itmSk.levelsArr[0]; assert freeSpace >= 0; if (freeSpace == 0) { @@ -325,6 +352,8 @@ static void updateItem(final KllItemsSketch itmSk, final Object item) { freeSpace = itmSk.levelsArr[0]; assert (freeSpace > 0); } + itmSk.setMinItem(newMin); + itmSk.setMaxItem(newMax); itmSk.incN(1); itmSk.setLevelZeroSorted(false); final int nextPos = freeSpace - 1; @@ -336,8 +365,7 @@ static void updateItem(final KllItemsSketch itmSk, final Object item) { static void updateItem(final KllItemsSketch itmSk, final T item, final long weight) { if (weight < itmSk.levelsArr[0]) { for (int i = 0; i < (int)weight; i++) { updateItem(itmSk, item); } - } else { - itmSk.updateMinMax(item); + } else { //the merge updates min and max final KllHeapItemsSketch tmpSk = new KllHeapItemsSketch<>(itmSk.getK(), DEFAULT_M, item, weight, itmSk.comparator, itmSk.serDe); itmSk.merge(tmpSk); diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java index 5c147b2c7..a39ad0823 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java @@ -378,10 +378,6 @@ int getTotalItemsNumBytes() { throw new SketchesArgumentException(UNSUPPORTED_MSG); } - @Override - void incNumLevels() { - //this is not used and must be a no-op. - } abstract void setItemsArray(Object[] ItemsArr); @@ -401,16 +397,6 @@ void setMemorySegment(final MemorySegment wseg) { throw new SketchesArgumentException(UNSUPPORTED_MSG + "Sketch not writable."); } - void updateMinMax(final T item) { - if (isEmpty()) { - setMinItem(item); - setMaxItem(item); - } else { - setMinItem(Util.minT(getMinItem(), item, comparator)); - setMaxItem(Util.maxT(getMaxItem(), item, comparator)); - } - } - private final ItemsSketchSortedView refreshSortedView() { if (itemsSV == null) { final CreateSortedView csv = new CreateSortedView(); diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java index 94638b047..3be9956a7 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java @@ -183,14 +183,7 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o myNewNumLevels = result[0]; assert myNewNumLevels <= ub; // ub may be much bigger - // THE NEW ITEMS ARRAY - myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length) - ? myCurLongItemsArray - : new long[targetItemCount]; final int freeSpaceAtBottom = targetItemCount - curItemCount; - - //shift the new items array create space at bottom - System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount); final int theShift = freeSpaceAtBottom - outlevels[0]; //calculate the new levels array length @@ -204,6 +197,16 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o myNewLevelsArr[lvl] = outlevels[lvl] + theShift; } + // THE NEW ITEMS ARRAY + //the levels array is allocated first, because this may be the current items array, + //and once it is overwritten nothing else may fail + myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length) + ? myCurLongItemsArray + : new long[targetItemCount]; + + //shift the new items array create space at bottom + System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount); + //MemorySegment SPACE MANAGEMENT if (mySketch.getMemorySegment() != null) { final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length); diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java index 22e6925a1..2c4312517 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -334,7 +334,6 @@ public void update(final long item) { //Also Called from KllLongsHelper::merge static void updateLong(final KllLongsSketch lngSk, final long item) { - lngSk.updateMinMax(item); int freeSpace = lngSk.levelsArr[0]; assert (freeSpace >= 0); if (freeSpace == 0) { @@ -342,6 +341,8 @@ static void updateLong(final KllLongsSketch lngSk, final long item) { freeSpace = lngSk.levelsArr[0]; assert (freeSpace > 0); } + //min and max change only after the compression, which may fail, has succeeded + lngSk.updateMinMax(item); lngSk.incN(1); lngSk.setLevelZeroSorted(false); final int nextPos = freeSpace - 1; diff --git a/src/main/java/org/apache/datasketches/kll/KllSketch.java b/src/main/java/org/apache/datasketches/kll/KllSketch.java index a91ee50e8..0ef42feff 100644 --- a/src/main/java/org/apache/datasketches/kll/KllSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllSketch.java @@ -89,6 +89,11 @@ * the storage up front instead. A sketch backed by a MemorySegment is not affected. * Results and the serialized form are the same either way.

* + *

Exceptions during update. If an update of a single item throws because storage cannot be + * allocated, because the {@code MemorySegmentRequest} of a sketch backed by a MemorySegment cannot provide a + * larger segment, or because the comparator of an items sketch throws, the item is not added and the sketch + * is unchanged. Weighted and vector updates, and merges, may have applied part of their input when they throw.

+ * * @see KLL Sketch * @see QuantilesAPI * @@ -451,8 +456,6 @@ final int getNumLevels() { abstract void incN(int increment); - abstract void incNumLevels(); - final boolean isCompactSingleItem() { return hasMemorySegment() && (sketchStructure == COMPACT_SINGLE) && (getN() == 1); } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java index 8db7857f5..8110729ed 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java @@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() { final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k); final KllDoublesSketch sk2 = KllDoublesSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setDoubleItemsArray(dblArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setDoubleItemsArrayAt(idx, dblV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java index 39798ae93..2bd37725a 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectFloatsSketchTest.java @@ -617,7 +617,7 @@ public void checkReadOnlyExceptions() { final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); final KllFloatsSketch sk2 = KllFloatsSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setFloatItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setFloatItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java index 277c3a403..96fcbc523 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java @@ -609,7 +609,7 @@ public void checkReadOnlyExceptions() { final KllLongsSketch sk = KllLongsSketch.newHeapInstance(k); final KllLongsSketch sk2 = KllLongsSketch.wrap(MemorySegment.ofArray(sk.toByteArray())); try { sk2.incN(1); fail(); } catch (final SketchesArgumentException e) { } - try { sk2.incNumLevels(); fail(); } catch (final SketchesArgumentException e) { } + try { sk2.setNumLevels(2); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLongItemsArray(fltArr); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLongItemsArrayAt(idx, fltV); fail(); } catch (final SketchesArgumentException e) { } try { sk2.setLevelZeroSorted(bool); fail(); } catch (final SketchesArgumentException e) { } diff --git a/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java new file mode 100644 index 000000000..cd199081c --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllUpdateFailureTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.MemorySegmentRequest; +import org.testng.annotations.Test; + +/** + * Checks that an update whose compaction fails leaves a KLL sketch consistent: + * the item is not added, and min and max are unchanged. + */ +public class KllUpdateFailureTest { + private static final int K = 20; + private static final long SEED = 12345; + + /** A MemorySegmentRequest that refuses every request. */ + private static final MemorySegmentRequest REFUSE = new MemorySegmentRequest() { + @Override + public MemorySegment request(final long newByteSize) { + throw new IllegalStateException("no more space"); + } + }; + + //A direct sketch whose segment cannot grow: the update that needs a new level fails, + //and the segment must still hold the sketch as it was before that update. + @Test + public void directDoublesRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllDoublesSketch.newHeapInstance(K), true)); + final KllDoublesSketch sk = KllDoublesSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(1000.0); //a new maximum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMaxItem(), K - 1.0); + final KllDoublesSketch copy = KllDoublesSketch.heapify(seg); + assertEquals(copy.getN(), K); + assertEquals(copy.getNumLevels(), 1); + assertEquals(copy.getMaxItem(), K - 1.0); + } + + @Test + public void directFloatsRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllFloatsSketch.newHeapInstance(K), true)); + final KllFloatsSketch sk = KllFloatsSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(-1000.0f); //a new minimum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMinItem(), 0.0f); + assertEquals(KllFloatsSketch.heapify(seg).getNumLevels(), 1); + } + + @Test + public void directLongsRefusedGrowthLeavesSegmentUnchanged() { + final MemorySegment seg = MemorySegment.ofArray(KllHelper.toByteArray(KllLongsSketch.newHeapInstance(K), true)); + final KllLongsSketch sk = KllLongsSketch.newDirectInstance(K, seg, REFUSE); + for (int i = 0; i < K; i++) { sk.update(i); } + final byte[] before = seg.toArray(JAVA_BYTE); + try { + sk.update(1000L); //a new maximum + fail("the segment request should have been refused"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(seg.toArray(JAVA_BYTE), before); + assertEquals(sk.getN(), K); + assertEquals(sk.getMaxItem(), K - 1L); + assertEquals(KllLongsSketch.heapify(seg).getNumLevels(), 1); + } + + /** A comparator that throws once a given number of further comparisons have been made. */ + private static final class FailingComparator implements Comparator { + private int allowed = -1; //negative: never throws + + void failAfter(final int comparisons) { allowed = comparisons; } + + @Override + public int compare(final String a, final String b) { + if (allowed == 0) { + allowed = -1; + throw new IllegalStateException("comparator failure"); + } + if (allowed > 0) { allowed--; } + return a.compareTo(b); + } + } + + private static String item(final int i) { return String.format("%06d", (i * 7919) % 100_003); } + + //The comparator throws while level 0 is sorted for compaction. The failed item is not added, + //min and max are unchanged, and after more updates the sketch matches one that never saw that item. + @Test + public void itemsComparatorFailureLeavesSketchConsistent() { + final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe(); + final int failAt = K; //the update that finds level 0 full + final int total = 50 * K; + + KllSketch.random.setSeed(SEED); + final FailingComparator comparator = new FailingComparator(); + final KllItemsSketch sk = KllItemsSketch.newHeapInstance(K, comparator, serDe); + for (int i = 0; i < total; i++) { + if (i == failAt) { + final String min = sk.getMinItem(); + final String max = sk.getMaxItem(); + comparator.failAfter(2); //the min and max checks pass, the sort fails + try { + sk.update("~"); //sorts after every item, so a new maximum + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(sk.getN(), failAt); + assertEquals(sk.getMinItem(), min); + assertEquals(sk.getMaxItem(), max); + } + sk.update(item(i)); + } + + KllSketch.random.setSeed(SEED); + final KllItemsSketch ref = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), serDe); + for (int i = 0; i < total; i++) { ref.update(item(i)); } + + assertEquals(sk.getN(), total); + assertEquals(sk.getMinItem(), ref.getMinItem()); + assertEquals(sk.getMaxItem(), ref.getMaxItem()); + assertEquals(sk.toByteArray(), ref.toByteArray()); + } +}