From bd0196ee351dd39e1308a1b5368c1c9f467cab89 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 18 Nov 2026 18:10:04 -0800 Subject: [PATCH] Quantiles items: keep the sketch consistent when an update throws update() counted the item in n and the base buffer count before processing a full base buffer. If processing threw, the base buffer count stayed at 2k: later updates then either threw ArrayIndexOutOfBoundsException or silently wrote over level 0. update() also stored a new maximum before the minimum comparison, which could throw. update() now compares first and commits n, the base buffer count, min and max only after processing succeeds, as HeapUpdateDoublesSketch does. The full base buffer is sorted as a copy, so a comparator that throws during the sort leaves the base buffer as it was. The merge code now allocates all the space it needs before it changes the target. Results and the serialized form are unchanged. The class javadoc states which failures leave the sketch unchanged; a comparator that throws while levels are merged, or is inconsistent, leaves it undefined. Co-Authored-By: Claude Opus 5.5 --- .../quantiles/ItemsMergeImpl.java | 34 +++-- .../datasketches/quantiles/ItemsUtil.java | 30 ++-- .../quantiles/QuantilesItemsSketch.java | 24 +++- .../quantiles/ItemsUpdateFailureTest.java | 128 ++++++++++++++++++ 4 files changed, 188 insertions(+), 28 deletions(-) create mode 100644 src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java diff --git a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java index 84d62dee6..dab3fc6e6 100644 --- a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java +++ b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java @@ -78,14 +78,14 @@ static void mergeInto(final QuantilesItemsSketch src, final QuantilesItem final Object[] srcCombBuf = src.getCombinedBuffer(); final long nFinal = tgtN + srcN; + //allocate everything first, so that an OutOfMemoryError leaves the target as it was + growForMerge(tgt, nFinal); + final Object[] scratchBuf = new Object[2 * tgtK]; + for (int i = 0; i < src.getBaseBufferCount(); i++) { //update only the base buffer tgt.update((T) srcCombBuf[i]); } - ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); - - final Object[] scratchBuf = new Object[2 * tgtK]; - long srcBitPattern = src.getBitPattern(); assert srcBitPattern == (srcN / (2L * srcK)); @@ -124,6 +124,22 @@ else if (tgtMin == null) { //if srcMin were null we would leave tgt alone } } + /** + * Grows the target's combined buffer to hold everything a merge giving nFinal items needs, + * so that the merge itself allocates nothing more in the target. + * @param the data type + * @param tgt the target sketch + * @param nFinal the number of items the target will have seen after the merge + */ + private static void growForMerge(final QuantilesItemsSketch tgt, final long nFinal) { + if (ClassicUtil.computeNumLevelsNeeded(tgt.getK(), nFinal) > 0) { + ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); + } else if (tgt.getCombinedBufferAllocatedCount() < nFinal) { //nFinal < 2k here + tgt.combinedBuffer_ = Arrays.copyOf(tgt.getCombinedBuffer(), (int) nFinal); + tgt.combinedBufferItemCapacity_ = (int) nFinal; + } + } + /** * Merges the source sketch into the target sketch that can have a smaller parameter K. * However, it is required that the ratio of the two K parameters be a power of 2. @@ -152,15 +168,15 @@ static void downSamplingMergeInto(final QuantilesItemsSketch src, final Q final long nFinal = tgt.getN() + src.getN(); + //allocate everything first, so that an OutOfMemoryError leaves the target as it was + growForMerge(tgt, nFinal); + final Object[] scratchBuf = new Object[2 * targetK]; + final Object[] downBuf = new Object[targetK]; + for (int i = 0; i < src.getBaseBufferCount(); i++) { tgt.update((T) sourceBaseBuffer[i]); } - ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal); - - final Object[] scratchBuf = new Object[2 * targetK]; - final Object[] downBuf = new Object[targetK]; - long srcBitPattern = src.getBitPattern(); for (int srcLvl = 0; srcBitPattern != 0L; srcLvl++, srcBitPattern >>>= 1) { if ((srcBitPattern & 1L) > 0L) { diff --git a/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java b/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java index 7ae7cec2f..383ebe360 100644 --- a/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java +++ b/src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java @@ -53,31 +53,35 @@ static void checkItemsSerVer(final int serVer) { } /** - * Called when the base buffer has just acquired 2*k elements. + * Called by update() when the new item, already placed in the base buffer but not yet counted, + * fills the base buffer to 2*k items. On success the base buffer is empty and the levels hold + * its items; the caller then updates n. If this throws, the counts and the levels are unchanged, + * unless the comparator throws while levels are being merged. * @param the data type * @param sketch the given quantiles sketch + * @param newN the number of items the sketch will have seen once the new item is counted */ @SuppressWarnings("unchecked") - static void processFullBaseBuffer(final QuantilesItemsSketch sketch) { - final int bbCount = sketch.getBaseBufferCount(); - final long n = sketch.getN(); - assert bbCount == (2 * sketch.getK()); // internal consistency check + static void processFullBaseBuffer(final QuantilesItemsSketch sketch, final long newN) { + final int k = sketch.getK(); + final int bbCount = 2 * k; + assert newN == (sketch.getN() + 1); // internal consistency check // make sure there will be enough levels for the propagation - ItemsUpdateImpl.maybeGrowLevels(sketch, n); // important: n_ was incremented by update before we got here + ItemsUpdateImpl.maybeGrowLevels(sketch, newN); - // this aliasing is a bit dangerous; notice that we did it after the possible resizing - final Object[] baseBuffer = sketch.getCombinedBuffer(); - - Arrays.sort((T[]) baseBuffer, 0, bbCount, sketch.getComparator()); + // sort a copy, so that a failing comparator leaves the base buffer as it was + final Object[] baseBuffer = sketch.getCombinedBuffer(); // after the possible resizing + final Object[] sortedBuffer = Arrays.copyOf(baseBuffer, bbCount); + Arrays.sort((T[]) sortedBuffer, 0, bbCount, sketch.getComparator()); ItemsUpdateImpl.inPlacePropagateCarry( 0, null, 0, // this null is okay - (T[]) baseBuffer, 0, + (T[]) sortedBuffer, 0, true, sketch); sketch.baseBufferCount_ = 0; - Arrays.fill(baseBuffer, 0, 2 * sketch.getK(), null); // to release the discarded objects - assert (n / (2L * sketch.getK())) == sketch.getBitPattern(); // internal consistency check + Arrays.fill(baseBuffer, 0, bbCount, null); // to release the discarded objects + assert (newN / (2L * k)) == sketch.getBitPattern(); // internal consistency check } static String toString(final boolean withLevels, final boolean withLevelsAndItems, diff --git a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java index f007ac392..d3ed300b9 100644 --- a/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java @@ -70,6 +70,12 @@ *

The size of an QuantilesItemsSketch is very dependent on the size of the generic Items input into the sketch, * so there is no comparable size table as there is for the QuantilesDoublesSketch.

* + *

Exceptions during update. If {@link #update(Object)} throws an {@code OutOfMemoryError}, + * the item is not added and the sketch is unchanged. The same holds if the comparator throws while + * the item is compared with the minimum and maximum items, or while the full base buffer is sorted. + * If the comparator throws while sorted levels are being merged, or does not impose a consistent + * total order, the state of the sketch is undefined.

+ * * @see QuantilesAPI * * @param The sketch item data type. @@ -541,17 +547,23 @@ public void update(final T item) { // this method only uses the base buffer part of the combined buffer if (item == null) { return; } - if ((maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0)) { maxItem_ = item; } - if ((minItem_ == null) || (comparator_.compare(item, minItem_) < 0)) { minItem_ = item; } + //compare first, but change nothing until the item has been fully processed, + //so that an exception leaves the sketch as it was + final boolean isNewMax = (maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0); + final boolean isNewMin = (minItem_ == null) || (comparator_.compare(item, minItem_) < 0); if ((baseBufferCount_ + 1) > combinedBufferItemCapacity_) { QuantilesItemsSketch.growBaseBuffer(this); } - combinedBuffer_[baseBufferCount_++] = item; - n_++; - if (baseBufferCount_ == (2 * k_)) { - ItemsUtil.processFullBaseBuffer(this); + combinedBuffer_[baseBufferCount_] = item; //not counted yet + if ((baseBufferCount_ + 1) == (2 * k_)) { + ItemsUtil.processFullBaseBuffer(this, n_ + 1); //empties the base buffer + } else { + baseBufferCount_++; } + n_++; + if (isNewMax) { maxItem_ = item; } + if (isNewMin) { minItem_ = item; } classicQisSV = null; } diff --git a/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java b/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java new file mode 100644 index 000000000..a6eba697c --- /dev/null +++ b/src/test/java/org/apache/datasketches/quantiles/ItemsUpdateFailureTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantiles; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfLongsSerDe; +import org.testng.annotations.Test; + +/** + * Checks that an update whose processing fails leaves a QuantilesItemsSketch as it was. + * The comparator is made to throw while the full base buffer is sorted. The failed item is not added, + * and the sketch must then match, byte for byte, a sketch that never saw that item. + */ +public class ItemsUpdateFailureTest { + private static final int K = 16; + private static final long SEED = 12345; + private static final ArrayOfLongsSerDe SERDE = new ArrayOfLongsSerDe(); + + /** A comparator that throws once a given number of further comparisons have been made. */ + private static final class FailingComparator implements Comparator { + private int allowed = -1; //negative: never throws + + void failAfter(final int comparisons) { allowed = comparisons; } + + @Override + public int compare(final Long a, final Long b) { + if (allowed == 0) { + allowed = -1; + throw new IllegalStateException("comparator failure"); + } + if (allowed > 0) { allowed--; } + return Long.compare(a, b); + } + } + + //Item i of the stream; the order is mixed so the sort does real work. + private static long item(final int i) { return ((i * 7919L) % 100_003L); } + + //The failing update is the one that fills the base buffer, so processing runs. + //At n = 2k the levels are grown before the sort fails; at n = 6k no growth is needed. + @Test + public void sortFailureLeavesSketchUnchanged() { + for (final int failAt : new int[] {(2 * K) - 1, (6 * K) - 1}) { + checkSortFailure(failAt, 20 * K); + } + } + + private static void checkSortFailure(final int failAt, final int total) { + QuantilesItemsSketch.rand.setSeed(SEED); + final FailingComparator comparator = new FailingComparator(); + final QuantilesItemsSketch sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator); + for (int i = 0; i < total; i++) { + if (i == failAt) { + comparator.failAfter(2); //the max and min checks pass, the sort fails + try { + sk.update(item(i)); + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(sk.getN(), failAt); + assertEquals(sk.getBaseBufferCount(), failAt % (2 * K)); + continue; + } + sk.update(item(i)); + } + + QuantilesItemsSketch.rand.setSeed(SEED); + final QuantilesItemsSketch ref = QuantilesItemsSketch.getInstance(Long.class, K, Comparator.naturalOrder()); + for (int i = 0; i < total; i++) { + if (i != failAt) { ref.update(item(i)); } + } + + assertEquals(sk.getN(), total - 1); + assertEquals(sk.getBitPattern(), ref.getBitPattern()); + assertEquals(sk.getBaseBufferCount(), ref.getBaseBufferCount()); + assertEquals(sk.getMinItem(), ref.getMinItem()); + assertEquals(sk.getMaxItem(), ref.getMaxItem()); + assertEquals(sk.toByteArray(SERDE), ref.toByteArray(SERDE)); + assertEquals(sk.getQuantile(0.5), ref.getQuantile(0.5)); + } + + //A comparator failure in the max or min check also leaves the sketch unchanged, + //including the max when only the min check fails. + @Test + public void minMaxFailureLeavesSketchUnchanged() { + final FailingComparator comparator = new FailingComparator(); + final QuantilesItemsSketch sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator); + sk.update(10L); + sk.update(20L); + comparator.failAfter(1); //the max check passes, the min check fails + try { + sk.update(30L); + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + assertEquals(sk.getN(), 2); + assertEquals(sk.getBaseBufferCount(), 2); + assertEquals(sk.getMaxItem(), Long.valueOf(20L)); + assertEquals(sk.getMinItem(), Long.valueOf(10L)); + sk.update(30L); + assertEquals(sk.getMaxItem(), Long.valueOf(30L)); + assertTrue(sk.getN() == 3); + } +}