Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,14 @@ static <T> void mergeInto(final QuantilesItemsSketch<T> src, final QuantilesItem
final Object[] srcCombBuf = src.getCombinedBuffer();
final long nFinal = tgtN + srcN;

//allocate everything first, so that an OutOfMemoryError leaves the target as it was
growForMerge(tgt, nFinal);
final Object[] scratchBuf = new Object[2 * tgtK];

for (int i = 0; i < src.getBaseBufferCount(); i++) { //update only the base buffer
tgt.update((T) srcCombBuf[i]);
}

ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal);

final Object[] scratchBuf = new Object[2 * tgtK];

long srcBitPattern = src.getBitPattern();
assert srcBitPattern == (srcN / (2L * srcK));

Expand Down Expand Up @@ -124,6 +124,22 @@ else if (tgtMin == null) { //if srcMin were null we would leave tgt alone
}
}

/**
* Grows the target's combined buffer to hold everything a merge giving nFinal items needs,
* so that the merge itself allocates nothing more in the target.
* @param <T> the data type
* @param tgt the target sketch
* @param nFinal the number of items the target will have seen after the merge
*/
private static <T> void growForMerge(final QuantilesItemsSketch<T> tgt, final long nFinal) {
if (ClassicUtil.computeNumLevelsNeeded(tgt.getK(), nFinal) > 0) {
ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal);
} else if (tgt.getCombinedBufferAllocatedCount() < nFinal) { //nFinal < 2k here
tgt.combinedBuffer_ = Arrays.copyOf(tgt.getCombinedBuffer(), (int) nFinal);
tgt.combinedBufferItemCapacity_ = (int) nFinal;
}
}

/**
* Merges the source sketch into the target sketch that can have a smaller parameter K.
* However, it is required that the ratio of the two K parameters be a power of 2.
Expand Down Expand Up @@ -152,15 +168,15 @@ static <T> void downSamplingMergeInto(final QuantilesItemsSketch<T> src, final Q

final long nFinal = tgt.getN() + src.getN();

//allocate everything first, so that an OutOfMemoryError leaves the target as it was
growForMerge(tgt, nFinal);
final Object[] scratchBuf = new Object[2 * targetK];
final Object[] downBuf = new Object[targetK];

for (int i = 0; i < src.getBaseBufferCount(); i++) {
tgt.update((T) sourceBaseBuffer[i]);
}

ItemsUpdateImpl.maybeGrowLevels(tgt, nFinal);

final Object[] scratchBuf = new Object[2 * targetK];
final Object[] downBuf = new Object[targetK];

long srcBitPattern = src.getBitPattern();
for (int srcLvl = 0; srcBitPattern != 0L; srcLvl++, srcBitPattern >>>= 1) {
if ((srcBitPattern & 1L) > 0L) {
Expand Down
30 changes: 17 additions & 13 deletions src/main/java/org/apache/datasketches/quantiles/ItemsUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,31 +53,35 @@ static void checkItemsSerVer(final int serVer) {
}

/**
* Called when the base buffer has just acquired 2*k elements.
* Called by update() when the new item, already placed in the base buffer but not yet counted,
* fills the base buffer to 2*k items. On success the base buffer is empty and the levels hold
* its items; the caller then updates n. If this throws, the counts and the levels are unchanged,
* unless the comparator throws while levels are being merged.
* @param <T> the data type
* @param sketch the given quantiles sketch
* @param newN the number of items the sketch will have seen once the new item is counted
*/
@SuppressWarnings("unchecked")
static <T> void processFullBaseBuffer(final QuantilesItemsSketch<T> sketch) {
final int bbCount = sketch.getBaseBufferCount();
final long n = sketch.getN();
assert bbCount == (2 * sketch.getK()); // internal consistency check
static <T> void processFullBaseBuffer(final QuantilesItemsSketch<T> sketch, final long newN) {
final int k = sketch.getK();
final int bbCount = 2 * k;
assert newN == (sketch.getN() + 1); // internal consistency check

// make sure there will be enough levels for the propagation
ItemsUpdateImpl.maybeGrowLevels(sketch, n); // important: n_ was incremented by update before we got here
ItemsUpdateImpl.maybeGrowLevels(sketch, newN);

// this aliasing is a bit dangerous; notice that we did it after the possible resizing
final Object[] baseBuffer = sketch.getCombinedBuffer();

Arrays.sort((T[]) baseBuffer, 0, bbCount, sketch.getComparator());
// sort a copy, so that a failing comparator leaves the base buffer as it was
final Object[] baseBuffer = sketch.getCombinedBuffer(); // after the possible resizing
final Object[] sortedBuffer = Arrays.copyOf(baseBuffer, bbCount);
Arrays.sort((T[]) sortedBuffer, 0, bbCount, sketch.getComparator());
ItemsUpdateImpl.inPlacePropagateCarry(
0,
null, 0, // this null is okay
(T[]) baseBuffer, 0,
(T[]) sortedBuffer, 0,
true, sketch);
sketch.baseBufferCount_ = 0;
Arrays.fill(baseBuffer, 0, 2 * sketch.getK(), null); // to release the discarded objects
assert (n / (2L * sketch.getK())) == sketch.getBitPattern(); // internal consistency check
Arrays.fill(baseBuffer, 0, bbCount, null); // to release the discarded objects
assert (newN / (2L * k)) == sketch.getBitPattern(); // internal consistency check
}

static <T> String toString(final boolean withLevels, final boolean withLevelsAndItems,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@
* <p>The size of an QuantilesItemsSketch is very dependent on the size of the generic Items input into the sketch,
* so there is no comparable size table as there is for the QuantilesDoublesSketch.</p>
*
* <p><b>Exceptions during update.</b> If {@link #update(Object)} throws an {@code OutOfMemoryError},
* the item is not added and the sketch is unchanged. The same holds if the comparator throws while
* the item is compared with the minimum and maximum items, or while the full base buffer is sorted.
* If the comparator throws while sorted levels are being merged, or does not impose a consistent
* total order, the state of the sketch is undefined.</p>
*
* @see QuantilesAPI
*
* @param <T> The sketch item data type.
Expand Down Expand Up @@ -541,17 +547,23 @@ public void update(final T item) {
// this method only uses the base buffer part of the combined buffer

if (item == null) { return; }
if ((maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0)) { maxItem_ = item; }
if ((minItem_ == null) || (comparator_.compare(item, minItem_) < 0)) { minItem_ = item; }
//compare first, but change nothing until the item has been fully processed,
//so that an exception leaves the sketch as it was
final boolean isNewMax = (maxItem_ == null) || (comparator_.compare(item, maxItem_) > 0);
final boolean isNewMin = (minItem_ == null) || (comparator_.compare(item, minItem_) < 0);

if ((baseBufferCount_ + 1) > combinedBufferItemCapacity_) {
QuantilesItemsSketch.growBaseBuffer(this);
}
combinedBuffer_[baseBufferCount_++] = item;
n_++;
if (baseBufferCount_ == (2 * k_)) {
ItemsUtil.processFullBaseBuffer(this);
combinedBuffer_[baseBufferCount_] = item; //not counted yet
if ((baseBufferCount_ + 1) == (2 * k_)) {
ItemsUtil.processFullBaseBuffer(this, n_ + 1); //empties the base buffer
} else {
baseBufferCount_++;
}
n_++;
if (isNewMax) { maxItem_ = item; }
if (isNewMin) { minItem_ = item; }
classicQisSV = null;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.quantiles;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

import java.util.Comparator;

import org.apache.datasketches.common.ArrayOfLongsSerDe;
import org.testng.annotations.Test;

/**
* Checks that an update whose processing fails leaves a QuantilesItemsSketch as it was.
* The comparator is made to throw while the full base buffer is sorted. The failed item is not added,
* and the sketch must then match, byte for byte, a sketch that never saw that item.
*/
public class ItemsUpdateFailureTest {
private static final int K = 16;
private static final long SEED = 12345;
private static final ArrayOfLongsSerDe SERDE = new ArrayOfLongsSerDe();

/** A comparator that throws once a given number of further comparisons have been made. */
private static final class FailingComparator implements Comparator<Long> {
private int allowed = -1; //negative: never throws

void failAfter(final int comparisons) { allowed = comparisons; }

@Override
public int compare(final Long a, final Long b) {
if (allowed == 0) {
allowed = -1;
throw new IllegalStateException("comparator failure");
}
if (allowed > 0) { allowed--; }
return Long.compare(a, b);
}
}

//Item i of the stream; the order is mixed so the sort does real work.
private static long item(final int i) { return ((i * 7919L) % 100_003L); }

//The failing update is the one that fills the base buffer, so processing runs.
//At n = 2k the levels are grown before the sort fails; at n = 6k no growth is needed.
@Test
public void sortFailureLeavesSketchUnchanged() {
for (final int failAt : new int[] {(2 * K) - 1, (6 * K) - 1}) {
checkSortFailure(failAt, 20 * K);
}
}

private static void checkSortFailure(final int failAt, final int total) {
QuantilesItemsSketch.rand.setSeed(SEED);
final FailingComparator comparator = new FailingComparator();
final QuantilesItemsSketch<Long> sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator);
for (int i = 0; i < total; i++) {
if (i == failAt) {
comparator.failAfter(2); //the max and min checks pass, the sort fails
try {
sk.update(item(i));
fail("the comparator should have thrown");
} catch (final IllegalStateException e) {
//expected
}
assertEquals(sk.getN(), failAt);
assertEquals(sk.getBaseBufferCount(), failAt % (2 * K));
continue;
}
sk.update(item(i));
}

QuantilesItemsSketch.rand.setSeed(SEED);
final QuantilesItemsSketch<Long> ref = QuantilesItemsSketch.getInstance(Long.class, K, Comparator.naturalOrder());
for (int i = 0; i < total; i++) {
if (i != failAt) { ref.update(item(i)); }
}

assertEquals(sk.getN(), total - 1);
assertEquals(sk.getBitPattern(), ref.getBitPattern());
assertEquals(sk.getBaseBufferCount(), ref.getBaseBufferCount());
assertEquals(sk.getMinItem(), ref.getMinItem());
assertEquals(sk.getMaxItem(), ref.getMaxItem());
assertEquals(sk.toByteArray(SERDE), ref.toByteArray(SERDE));
assertEquals(sk.getQuantile(0.5), ref.getQuantile(0.5));
}

//A comparator failure in the max or min check also leaves the sketch unchanged,
//including the max when only the min check fails.
@Test
public void minMaxFailureLeavesSketchUnchanged() {
final FailingComparator comparator = new FailingComparator();
final QuantilesItemsSketch<Long> sk = QuantilesItemsSketch.getInstance(Long.class, K, comparator);
sk.update(10L);
sk.update(20L);
comparator.failAfter(1); //the max check passes, the min check fails
try {
sk.update(30L);
fail("the comparator should have thrown");
} catch (final IllegalStateException e) {
//expected
}
assertEquals(sk.getN(), 2);
assertEquals(sk.getBaseBufferCount(), 2);
assertEquals(sk.getMaxItem(), Long.valueOf(20L));
assertEquals(sk.getMinItem(), Long.valueOf(10L));
sk.update(30L);
assertEquals(sk.getMaxItem(), Long.valueOf(30L));
assertTrue(sk.getN() == 3);
}
}
Loading