diff --git a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java index 84d62dee6..dab3fc6e6 100644 --- a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java +++ b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java @@ -78,14 +78,14 @@ static void mergeInto(final QuantilesItemsSketch src, final QuantilesItem final Object[] srcCombBuf = src.getCombinedBuffer(); final long nFinal = tgtN + srcN; + //allocate everything first, so that an OutOfMemoryError leaves the target as it was + growForMerge(tgt, nFinal); + final Object[] scratchBuf = new Object[2 * tgtK]; + for (int i = 0; i < src.getBaseBufferCount(); i++) { //update only the base buffer tgt.update((T) srcCombBuf[i]); } - ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); - - final Object[] scratchBuf = new Object[2 * tgtK]; - long srcBitPattern = src.getBitPattern(); assert srcBitPattern == (srcN / (2L * srcK)); @@ -124,6 +124,22 @@ else if (tgtMin == null) { //if srcMin were null we would leave tgt alone } } + /** + * Grows the target's combined buffer to hold everything a merge giving nFinal items needs, + * so that the merge itself allocates nothing more in the target. + * @param the data type + * @param tgt the target sketch + * @param nFinal the number of items the target will have seen after the merge + */ + private static void growForMerge(final QuantilesItemsSketch tgt, final long nFinal) { + if (ClassicUtil.computeNumLevelsNeeded(tgt.getK(), nFinal) > 0) { + ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); + } else if (tgt.getCombinedBufferAllocatedCount() < nFinal) { //nFinal < 2k here + tgt.combinedBuffer_ = Arrays.copyOf(tgt.getCombinedBuffer(), (int) nFinal); + tgt.combinedBufferItemCapacity_ = (int) nFinal; + } + } + /** * Merges the source sketch into the target sketch that can have a smaller parameter K. * However, it is required that the ratio of the two K parameters be a power of 2. @@ -152,15 +168,15 @@ static void downSamplingMergeInto(final QuantilesItemsSketch src, final Q final long nFinal = tgt.getN() + src.getN(); + //allocate everything first, so that an OutOfMemoryError leaves the target as it was + growForMerge(tgt, nFinal); + final Object[] scratchBuf = new Object[2 * targetK]; + final Object[] downBuf = new Object[targetK]; + for (int i = 0; i < src.getBaseBufferCount(); i++) { tgt.update((T) sourceBaseBuffer[i]); } - ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); - - final Object[] scratchBuf = new Object[2 * targetK]; - final Object[] downBuf = new Object[targetK]; - long srcBitPattern = src.getBitPattern(); for (int srcLvl = 0; srcBitPattern != 0L; srcLvl++, srcBitPattern >>>= 1) { if ((srcBitPattern & 1L) > 0L) { diff --git a/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java b/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java index 7ae7cec2f..383ebe360 100644 --- a/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java +++ b/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java @@ -53,31 +53,35 @@ static void checkItemsSerVer(final int serVer) { } /** - * Called when the base buffer has just acquired 2*k elements. + * Called by update() when the new item, already placed in the base buffer but not yet counted, + * fills the base buffer to 2*k items. On success the base buffer is empty and the levels hold + * its items; the caller then updates n. If this throws, the counts and the levels are unchanged, + * unless the comparator throws while levels are being merged. * @param the data type * @param sketch the given quantiles sketch + * @param newN the number of items the sketch will have seen once the new item is counted */ @SuppressWarnings("unchecked") - static void processFullBaseBuffer(final QuantilesItemsSketch sketch) { - final int bbCount = sketch.getBaseBufferCount(); - final long n = sketch.getN(); - assert bbCount == (2 * sketch.getK()); // internal consistency check + static void processFullBaseBuffer(final QuantilesItemsSketch sketch, final long newN) { + final int k = sketch.getK(); + final int bbCount = 2 * k; + assert newN == (sketch.getN() + 1); // internal consistency check // make sure there will be enough levels for the propagation - ItemsUpdateImpl.maybeGrowLevels(sketch, n); // important: n_ was incremented by update before we got here + ItemsUpdateImpl.maybeGrowLevels(sketch, newN); - // this aliasing is a bit dangerous; notice that we did it after the possible resizing - final Object[] baseBuffer = sketch.getCombinedBuffer(); - - Arrays.sort((T[]) baseBuffer, 0, bbCount, sketch.getComparator()); + // sort a copy, so that a failing comparator leaves the base buffer as it was + final Object[] baseBuffer = sketch.getCombinedBuffer(); // after the possible resizing + final Object[] sortedBuffer = Arrays.copyOf(baseBuffer, bbCount); + Arrays.sort((T[]) sortedBuffer, 0, bbCount, sketch.getComparator()); ItemsUpdateImpl.inPlacePropagateCarry( 0, null, 0, // this null is okay - (T[]) baseBuffer, 0, + (T[]) sortedBuffer, 0, true, sketch); sketch.baseBufferCount_ = 0; - Arrays.fill(baseBuffer, 0, 2 * sketch.getK(), null); // to release the discarded objects - assert (n / (2L * sketch.getK())) == sketch.getBitPattern(); // internal consistency check + Arrays.fill(baseBuffer, 0, bbCount, null); // to release the discarded objects + assert (newN / (2L * k)) == sketch.getBitPattern(); // internal consistency check } static String toString(final boolean withLevels, final boolean withLevelsAndItems, diff --git a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java index f007ac392..d3ed300b9 100644 --- a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java @@ -70,6 +70,12 @@ *

The size of an QuantilesItemsSketch is very dependent on the size of the generic Items input into the sketch, * so there is no comparable size table as there is for the QuantilesDoublesSketch.

* + *

Exceptions during update. If {@link #update(Object)} throws an {@code OutOfMemoryError}, + * the item is not added and the sketch is unchanged. The same holds if the comparator throws while + * the item is compared with the minimum and maximum items, or while the full base buffer is sorted. + * If the comparator throws while sorted levels are being merged, or does not impose a consistent + * total order, the state of the sketch is undefined.

+ * * @see QuantilesAPI * * @param The sketch item data type. @@ -541,17 +547,23 @@ public void update(final T item) { // this method only uses the base buffer part of the combined buffer if (item == null) { return; } - if ((maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0)) { maxItem_ = item; } - if ((minItem_ == null) || (comparator_.compare(item, minItem_) < 0)) { minItem_ = item; } + //compare first, but change nothing until the item has been fully processed, + //so that an exception leaves the sketch as it was + final boolean isNewMax = (maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0); + final boolean isNewMin = (minItem_ == null) || (comparator_.compare(item, minItem_) < 0); if ((baseBufferCount_ + 1) > combinedBufferItemCapacity_) { QuantilesItemsSketch.growBaseBuffer(this); } - combinedBuffer_[baseBufferCount_++] = item; - n_++; - if (baseBufferCount_ == (2 * k_)) { - ItemsUtil.processFullBaseBuffer(this); + combinedBuffer_[baseBufferCount_] = item; //not counted yet + if ((baseBufferCount_ + 1) == (2 * k_)) { + ItemsUtil.processFullBaseBuffer(this, n_ + 1); //empties the base buffer + } else { + baseBufferCount_++; } + n_++; + if (isNewMax) { maxItem_ = item; } + if (isNewMin) { minItem_ = item; } classicQisSV = null; } diff --git a/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java b/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java new file mode 100644 index 000000000..a6eba697c --- /dev/null +++ b/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantiles; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfLongsSerDe; +import org.testng.annotations.Test; + +/** + * Checks that an update whose processing fails leaves a QuantilesItemsSketch as it was. + * The comparator is made to throw while the full base buffer is sorted. The failed item is not added, + * and the sketch must then match, byte for byte, a sketch that never saw that item. + */ +public class ItemsUpdateFailureTest { + private static final int K = 16; + private static final long SEED = 12345; + private static final ArrayOfLongsSerDe SERDE = new ArrayOfLongsSerDe(); + + /** A comparator that throws once a given number of further comparisons have been made. */ + private static final class FailingComparator implements Comparator { + private int allowed = -1; //negative: never throws + + void failAfter(final int comparisons) { allowed = comparisons; } + + @Override + public int compare(final Long a, final Long b) { + if (allowed == 0) { + allowed = -1; + throw new IllegalStateException("comparator failure"); + } + if (allowed > 0) { allowed--; } + return Long.compare(a, b); + } + } + + //Item i of the stream; the order is mixed so the sort does real work. + private static long item(final int i) { return ((i * 7919L) % 100_003L); } + + //The failing update is the one that fills the base buffer, so processing runs. + //At n = 2k the levels are grown before the sort fails; at n = 6k no growth is needed. + @Test + public void sortFailureLeavesSketchUnchanged() { + for (final int failAt : new int[] {(2 * K) - 1, (6 * K) - 1}) { + checkSortFailure(failAt, 20 * K); + } + } + + private static void checkSortFailure(final int failAt, final int total) { + QuantilesItemsSketch.rand.setSeed(SEED); + final FailingComparator comparator = new FailingComparator(); + final QuantilesItemsSketch sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator); + for (int i = 0; i < total; i++) { + if (i == failAt) { + comparator.failAfter(2); //the max and min checks pass, the sort fails + try { + sk.update(item(i)); + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(sk.getN(), failAt); + assertEquals(sk.getBaseBufferCount(), failAt % (2 * K)); + continue; + } + sk.update(item(i)); + } + + QuantilesItemsSketch.rand.setSeed(SEED); + final QuantilesItemsSketch ref = QuantilesItemsSketch.getInstance(Long.class, K, Comparator.naturalOrder()); + for (int i = 0; i < total; i++) { + if (i != failAt) { ref.update(item(i)); } + } + + assertEquals(sk.getN(), total - 1); + assertEquals(sk.getBitPattern(), ref.getBitPattern()); + assertEquals(sk.getBaseBufferCount(), ref.getBaseBufferCount()); + assertEquals(sk.getMinItem(), ref.getMinItem()); + assertEquals(sk.getMaxItem(), ref.getMaxItem()); + assertEquals(sk.toByteArray(SERDE), ref.toByteArray(SERDE)); + assertEquals(sk.getQuantile(0.5), ref.getQuantile(0.5)); + } + + //A comparator failure in the max or min check also leaves the sketch unchanged, + //including the max when only the min check fails. + @Test + public void minMaxFailureLeavesSketchUnchanged() { + final FailingComparator comparator = new FailingComparator(); + final QuantilesItemsSketch sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator); + sk.update(10L); + sk.update(20L); + comparator.failAfter(1); //the max check passes, the min check fails + try { + sk.update(30L); + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(sk.getN(), 2); + assertEquals(sk.getBaseBufferCount(), 2); + assertEquals(sk.getMaxItem(), Long.valueOf(20L)); + assertEquals(sk.getMinItem(), Long.valueOf(10L)); + sk.update(30L); + assertEquals(sk.getMaxItem(), Long.valueOf(30L)); + assertTrue(sk.getN() == 3); + } +}