The default k of 200 yields a "single-sided" epsilon of about 1.33% and a * "double-sided" (PMF) epsilon of about 1.65%, with a confidence of 99%.
* + *Memory allocation. A new heap sketch does not allocate storage for items until it is first used: + * the first update, or the first merge of a non-empty sketch. After that, the storage grows as the sketch + * needs it. An {@link OutOfMemoryError} from that first allocation therefore happens in the update or merge, + * not when the sketch is created. Call {@link #ensureCapacity()} right after creating the sketch to allocate + * the storage up front instead. A sketch backed by a MemorySegment is not affected. + * Results and the serialized form are the same either way.
+ * * @see KLL Sketch * @see QuantilesAPI * @@ -269,6 +276,16 @@ public final boolean isReadOnly() { */ public abstract void merge(KllSketch other); + /** + * Allocates the item storage of a heap sketch now, rather than on its first update or merge. + * The storage is the initial capacity of k items. Call this right after creating the sketch to move + * an {@link OutOfMemoryError} from the first update or merge to this call. It has no effect if the storage + * already exists, and none for a sketch backed by a MemorySegment. + * + *{@link #reset()} releases the storage of a heap sketch, and it is allocated again on first use.
+ */ + public void ensureCapacity() { } + @Override public final String toString() { return toString(false, false); diff --git a/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java new file mode 100644 index 000000000..875d6cc7c --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; +import java.util.List; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.testng.annotations.Test; + +/** + * Checks that heap KLL sketches allocate item storage on first use, or when ensureCapacity() is called. + * KLL compaction is random, so images are compared byte for byte only below k items, before any compaction. + */ +public class KllLazyAllocationTest { + private static final int K = 200; + private static final ArrayOfStringsSerDe SERDE = new ArrayOfStringsSerDe(); + + private static KllHeapDoublesSketch heapDoubles(final KllDoublesSketch sk) { return (KllHeapDoublesSketch) sk; } + + @Test + public void doublesEmptySketchHoldsNoStorage() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + assertFalse(heapDoubles(sk).hasItemStorage()); + sk.update(Double.NaN); //ignored, so not a first use + assertTrue(sk.isEmpty()); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertFalse(sk.iterator().next()); + sk.toString(true, true); + + final KllDoublesSketch ref = KllDoublesSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + + sk.merge(KllDoublesSketch.newHeapInstance(K)); //merging an empty sketch is not a first use + assertFalse(heapDoubles(sk).hasItemStorage()); + + final KllDoublesSketch copy = KllDoublesSketch.heapify(MemorySegment.ofArray(sk.toByteArray())); + assertFalse(heapDoubles(copy).hasItemStorage()); + assertTrue(copy.isEmpty()); + assertFalse(heapDoubles(sk).hasItemStorage()); + assertFalse(heapDoubles(copy).hasItemStorage()); + } + + @Test + public void doublesEnsureCapacity() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + sk.ensureCapacity(); + final KllHeapDoublesSketch heap = heapDoubles(sk); + assertTrue(heap.hasItemStorage()); + final double[] storage = heap.getDoubleItemsArray(); + assertEquals(storage.length, K); + sk.ensureCapacity(); //no effect once allocated + assertSame(heap.getDoubleItemsArray(), storage); + + final KllDoublesSketch lazy = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < K; i++) { sk.update(i); lazy.update(i); } //fills the initial storage, no compaction + assertSame(heap.getDoubleItemsArray(), storage); //nothing more was allocated + assertEquals(sk.toByteArray(), lazy.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(lazy, true)); + + final double[] before = heap.getDoubleItemsArray(); + final byte[] image = sk.toByteArray(); + sk.ensureCapacity(); //no effect after updates + assertSame(heap.getDoubleItemsArray(), before); + assertEquals(sk.toByteArray(), image); + } + + @Test + public void doublesFirstUseAllocates() { + final KllDoublesSketch single = KllDoublesSketch.newHeapInstance(K); + single.update(1.0); + assertTrue(heapDoubles(single).hasItemStorage()); + assertEquals(single.getMinItem(), 1.0); + + final KllDoublesSketch weighted = KllDoublesSketch.newHeapInstance(K); + weighted.update(2.0, 3); + assertTrue(heapDoubles(weighted).hasItemStorage()); + assertEquals(weighted.getN(), 3); + + final KllDoublesSketch heavy = KllDoublesSketch.newHeapInstance(K); + heavy.update(2.0, 10_000); //takes the merge path + assertEquals(heavy.getN(), 10_000); + assertEquals(heavy.getRank(2.0, INCLUSIVE), 1.0); + + final KllDoublesSketch vector = KllDoublesSketch.newHeapInstance(K); + vector.update(new double[] {3.0, Double.NaN, 4.0}, 0, 3); + assertEquals(vector.getN(), 2); + assertEquals(vector.getMaxItem(), 4.0); + + final KllDoublesSketch merged = KllDoublesSketch.newHeapInstance(K); + final KllDoublesSketch other = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 10_000; i++) { other.update(i); } + merged.merge(other); + assertTrue(heapDoubles(merged).hasItemStorage()); + assertEquals(merged.getN(), 10_000); + assertEquals(merged.getMinItem(), 0.0); + assertEquals(merged.getMaxItem(), 9999.0); + final double r = merged.getRank(5000, INCLUSIVE); + assertTrue(Math.abs(r - 0.5) < 0.05, "rank " + r); + + final KllDoublesSketch fromSingle = KllDoublesSketch.newHeapInstance(K); + fromSingle.merge(KllDoublesSketch.heapify(MemorySegment.ofArray(single.toByteArray()))); + assertEquals(fromSingle.toByteArray(), single.toByteArray()); + } + + @Test + public void doublesResetReleasesStorage() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 1000; i++) { sk.update(i); } + sk.reset(); + assertFalse(heapDoubles(sk).hasItemStorage()); + assertTrue(sk.isEmpty()); + final KllDoublesSketch fresh = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 100; i++) { sk.update(i); fresh.update(i); } + assertEquals(sk.toByteArray(), fresh.toByteArray()); + } + + @Test + public void directSketchIsUnaffected() { + final int bytes = KllSketch.getMaxSerializedSizeBytes(K, 1000, KllSketch.SketchType.KLL_DOUBLES_SKETCH, true); + final KllDoublesSketch sk = KllDoublesSketch.newDirectInstance(K, MemorySegment.ofArray(new byte[bytes]), null); + sk.ensureCapacity(); //no effect + for (int i = 0; i < 1000; i++) { sk.update(i); } + sk.reset(); + assertTrue(sk.isEmpty()); + sk.update(1.0); + assertEquals(sk.getN(), 1); + } + + @Test + public void floatsLazy() { + final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(K); + final KllHeapFloatsSketch heap = (KllHeapFloatsSketch) sk; + assertFalse(heap.hasItemStorage()); + sk.update(Float.NaN); + assertFalse(heap.hasItemStorage()); + final KllFloatsSketch ref = KllFloatsSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertEquals(heap.getFloatItemsArray().length, K); //reading the array for writing allocates it + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + + final KllFloatsSketch lazy = KllFloatsSketch.newHeapInstance(K); + assertFalse(((KllHeapFloatsSketch) lazy).hasItemStorage()); + for (int i = 0; i < K; i++) { ref.update(i); lazy.update(i); } + assertEquals(ref.toByteArray(), lazy.toByteArray()); + lazy.reset(); + assertFalse(((KllHeapFloatsSketch) lazy).hasItemStorage()); + final KllFloatsSketch merged = KllFloatsSketch.newHeapInstance(K); + merged.merge(ref); + assertEquals(merged.getN(), K); + } + + @Test + public void longsLazy() { + final KllLongsSketch sk = KllLongsSketch.newHeapInstance(K); + final KllHeapLongsSketch heap = (KllHeapLongsSketch) sk; + assertFalse(heap.hasItemStorage()); + final KllLongsSketch ref = KllLongsSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertTrue(((KllHeapLongsSketch) ref).hasItemStorage()); + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + assertFalse(heap.hasItemStorage()); + + for (int i = 0; i < K; i++) { sk.update(i); ref.update(i); } + assertEquals(sk.toByteArray(), ref.toByteArray()); + sk.reset(); + assertFalse(heap.hasItemStorage()); + final KllLongsSketch merged = KllLongsSketch.newHeapInstance(K); + merged.merge(ref); + assertEquals(merged.getN(), K); + assertEquals(merged.getMaxItem(), K - 1); + } + + @Test + public void itemsLazy() { + final KllItemsSketch