From efcb8830e3879e2d6c17a9c9abefbd5e546b55aa Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 12 Nov 2026 11:55:38 -0800 Subject: [PATCH 1/2] KLL: allocate heap item storage on first use; add ensureCapacity() Heap KLL sketches (doubles, floats, longs and items) no longer allocate item storage when they are created, when an empty image is heapified, or on reset(). The storage is allocated on the first update, or on the first merge of a non-empty sketch, before anything else changes. ensureCapacity() allocates it up front. It has no effect once the storage exists, nor for sketches backed by a MemorySegment. This follows the lazy allocation convention agreed on dev@. Results and the serialized form are unchanged. An OutOfMemoryError from the first allocation now comes from the first update or merge instead of the constructor; the class javadoc says so. Co-Authored-By: Claude Opus 5.5 --- .../datasketches/kll/KllDoublesSketch.java | 11 +- .../datasketches/kll/KllFloatsSketch.java | 11 +- .../kll/KllHeapDoublesSketch.java | 29 +- .../datasketches/kll/KllHeapFloatsSketch.java | 29 +- .../datasketches/kll/KllHeapItemsSketch.java | 27 +- .../datasketches/kll/KllHeapLongsSketch.java | 29 +- .../datasketches/kll/KllItemsSketch.java | 8 +- .../datasketches/kll/KllLongsSketch.java | 11 +- .../apache/datasketches/kll/KllSketch.java | 17 ++ .../kll/KllLazyAllocationTest.java | 247 ++++++++++++++++++ 10 files changed, 395 insertions(+), 24 deletions(-) create mode 100644 src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java index 2981d3628..9adaca31f 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java @@ -68,6 +68,7 @@ public abstract class KllDoublesSketch extends KllSketch implements QuantilesDou * Create a new heap instance of this sketch with the default k = 200. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @return new KllDoublesSketch on the Java heap. */ public static KllDoublesSketch newHeapInstance() { @@ -79,6 +80,7 @@ public static KllDoublesSketch newHeapInstance() { * k can be between 8, inclusive, and 65535, inclusive. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @param k parameter that controls size of the sketch and accuracy of estimates. * @return new KllDoublesSketch on the Java heap. */ @@ -268,7 +270,7 @@ public double[] getRanks(final double[] quantiles, final QuantileSearchCriteria @Override public QuantilesDoublesSketchIteratorAPI iterator() { return new KllDoublesSketchIterator( - getDoubleItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); + isEmpty() ? new double[0] : getDoubleItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); } @Override @@ -277,6 +279,7 @@ public final void merge(final KllSketch other) { if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } final KllDoublesSketch othDblSk = (KllDoublesSketch)other; if (othDblSk.isEmpty()) { return; } + ensureCapacity(); //first use allocates, before anything changes KllDoublesHelper.mergeDoubleImpl(this, othDblSk); doublesSV = null; } @@ -297,7 +300,8 @@ public final void reset() { setLevelsArray(new int[] {k, k}); setMinItem(Double.NaN); setMaxItem(Double.NaN); - setDoubleItemsArray(new double[k]); + //a heap sketch releases its storage and allocates it again on first use + setDoubleItemsArray(hasMemorySegment() ? new double[k] : null); doublesSV = null; } @@ -323,6 +327,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem public void update(final double item) { if (Double.isNaN(item)) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + ensureCapacity(); //first use allocates, before anything changes updateDouble(this, item); doublesSV = null; } @@ -369,6 +374,7 @@ public void update(final double item, final long weight) { if (Double.isNaN(item)) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + ensureCapacity(); //first use allocates, before anything changes if (weight == 1L) { updateDouble(this, item); } else if (weight < levelsArr[0]) { for (int i = 0; i < (int)weight; i++) { updateDouble(this, item); } @@ -393,6 +399,7 @@ else if (weight < levelsArr[0]) { public void update(final double[] items, final int offset, final int length) { if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (length == 0) { return; } + ensureCapacity(); //first use allocates, before anything changes if (!hasNaN(items, offset, length)) { updateDouble(items, offset, length); //fast path doublesSV = null; diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java index b2741047c..90e3da5fe 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java @@ -68,6 +68,7 @@ public abstract class KllFloatsSketch extends KllSketch implements QuantilesFloa * Create a new heap instance of this sketch with the default k = 200. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @return new KllFloatsSketch on the Java heap. */ public static KllFloatsSketch newHeapInstance() { @@ -79,6 +80,7 @@ public static KllFloatsSketch newHeapInstance() { * k can be between 8, inclusive, and 65535, inclusive. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @param k parameter that controls size of the sketch and accuracy of estimates. * @return new KllFloatsSketch on the Java heap. */ @@ -268,7 +270,7 @@ public double[] getRanks(final float[] quantiles, final QuantileSearchCriteria s @Override public QuantilesFloatsSketchIterator iterator() { return new KllFloatsSketchIterator( - getFloatItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); + isEmpty() ? new float[0] : getFloatItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); } @Override @@ -277,6 +279,7 @@ public final void merge(final KllSketch other) { if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } final KllFloatsSketch othFltSk = (KllFloatsSketch)other; if (othFltSk.isEmpty()) { return; } + ensureCapacity(); //first use allocates, before anything changes KllFloatsHelper.mergeFloatImpl(this, othFltSk); floatsSV = null; } @@ -297,7 +300,8 @@ public final void reset() { setLevelsArray(new int[] {k, k}); setMinItem(Float.NaN); setMaxItem(Float.NaN); - setFloatItemsArray(new float[k]); + //a heap sketch releases its storage and allocates it again on first use + setFloatItemsArray(hasMemorySegment() ? new float[k] : null); floatsSV = null; } @@ -323,6 +327,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem public void update(final float item) { if (Float.isNaN(item)) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + ensureCapacity(); //first use allocates, before anything changes updateFloat(this, item); floatsSV = null; } @@ -369,6 +374,7 @@ public void update(final float item, final long weight) { if (Float.isNaN(item)) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + ensureCapacity(); //first use allocates, before anything changes if (weight == 1L) { updateFloat(this, item); } else if (weight < levelsArr[0]) { for (int i = 0; i < (int)weight; i++) { updateFloat(this, item); } @@ -393,6 +399,7 @@ else if (weight < levelsArr[0]) { public void update(final float[] items, final int offset, final int length) { if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (length == 0) { return; } + ensureCapacity(); //first use allocates, before anything changes if (!hasNaN(items, offset, length)) { updateFloat(items, offset, length); //fast path floatsSV = null; diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java index c6de1a5d8..dc82e0796 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java @@ -52,7 +52,7 @@ final class KllHeapDoublesSketch extends KllDoublesSketch { private boolean isLevelZeroSorted; private double minDoubleItem; private double maxDoubleItem; - private double[] doubleItems; + private double[] doubleItems; // null until first use, see ensureCapacity() /** * New instance heap constructor with a given parameters k and m. @@ -76,7 +76,7 @@ final class KllHeapDoublesSketch extends KllDoublesSketch { isLevelZeroSorted = false; minDoubleItem = Double.NaN; maxDoubleItem = Double.NaN; - doubleItems = new double[k]; + doubleItems = null; //allocated on first use, or by ensureCapacity() } /** @@ -118,7 +118,7 @@ private KllHeapDoublesSketch( if (segStructure == COMPACT_EMPTY) { minDoubleItem = Double.NaN; maxDoubleItem = Double.NaN; - doubleItems = new double[k]; + doubleItems = null; //allocated on first use } else if (segStructure == COMPACT_SINGLE) { final double item = srcSeg.get(JAVA_DOUBLE_UNALIGNED, DATA_START_ADR_SINGLE_ITEM); @@ -221,7 +221,26 @@ byte[] getMinMaxByteArr() { //other restricted @Override - double[] getDoubleItemsArray() { return doubleItems; } + double[] getDoubleItemsArray() { + ensureCapacity(); //callers may write into the returned array + return doubleItems; + } + + /** + * Allocates the item storage now, which a heap sketch otherwise allocates on first use. + * It has no effect if the storage already exists. + * @see KllSketch#ensureCapacity() + */ + @Override + public void ensureCapacity() { + if (doubleItems == null) { doubleItems = new double[k]; } + } + + /** + * Returns true if the item storage has been allocated. + * @return true if the item storage has been allocated + */ + boolean hasItemStorage() { return doubleItems != null; } @Override double getDoubleSingleItem() { @@ -254,6 +273,7 @@ byte[] getRetainedItemsByteArr() { @Override byte[] getTotalItemsByteArr() { + if (doubleItems == null) { return new byte[k * Double.BYTES]; } //empty, all zeros final byte[] byteArr = new byte[doubleItems.length * Double.BYTES]; final MemorySegment wseg = MemorySegment.ofArray(byteArr); MemorySegment.copy(doubleItems, 0, wseg, JAVA_DOUBLE_UNALIGNED, 0, doubleItems.length); @@ -303,6 +323,7 @@ void setNumLevels(final int numLevels) { @Override double[] getDoubleRetainedItemsArray() { + if (doubleItems == null) { return new double[0]; } return Arrays.copyOfRange(doubleItems, levelsArr[0], levelsArr[getNumLevels()]); } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java index 55b990d3c..becba9a40 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java @@ -52,7 +52,7 @@ final class KllHeapFloatsSketch extends KllFloatsSketch { private boolean isLevelZeroSorted; private float minFloatItem; private float maxFloatItem; - private float[] floatItems; + private float[] floatItems; // null until first use, see ensureCapacity() /** * New instance heap constructor with a given parameters k and m. @@ -76,7 +76,7 @@ final class KllHeapFloatsSketch extends KllFloatsSketch { isLevelZeroSorted = false; minFloatItem = Float.NaN; maxFloatItem = Float.NaN; - floatItems = new float[k]; + floatItems = null; //allocated on first use, or by ensureCapacity() } /** @@ -118,7 +118,7 @@ private KllHeapFloatsSketch( if (segStructure == COMPACT_EMPTY) { minFloatItem = Float.NaN; maxFloatItem = Float.NaN; - floatItems = new float[k]; + floatItems = null; //allocated on first use } else if (segStructure == COMPACT_SINGLE) { final float item = srcSeg.get(JAVA_FLOAT_UNALIGNED, DATA_START_ADR_SINGLE_ITEM); @@ -221,7 +221,26 @@ byte[] getMinMaxByteArr() { //other restricted @Override - float[] getFloatItemsArray() { return floatItems; } + float[] getFloatItemsArray() { + ensureCapacity(); //callers may write into the returned array + return floatItems; + } + + /** + * Allocates the item storage now, which a heap sketch otherwise allocates on first use. + * It has no effect if the storage already exists. + * @see KllSketch#ensureCapacity() + */ + @Override + public void ensureCapacity() { + if (floatItems == null) { floatItems = new float[k]; } + } + + /** + * Returns true if the item storage has been allocated. + * @return true if the item storage has been allocated + */ + boolean hasItemStorage() { return floatItems != null; } @Override float getFloatSingleItem() { @@ -254,6 +273,7 @@ byte[] getRetainedItemsByteArr() { @Override byte[] getTotalItemsByteArr() { + if (floatItems == null) { return new byte[k * Float.BYTES]; } //empty, all zeros final byte[] byteArr = new byte[floatItems.length * Float.BYTES]; final MemorySegment wseg = MemorySegment.ofArray(byteArr); MemorySegment.copy(floatItems, 0, wseg, JAVA_FLOAT_UNALIGNED, 0, floatItems.length); @@ -303,6 +323,7 @@ void setNumLevels(final int numLevels) { @Override float[] getFloatRetainedItemsArray() { + if (floatItems == null) { return new float[0]; } return Arrays.copyOfRange(floatItems, levelsArr[0], levelsArr[getNumLevels()]); } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java index 74640e902..9d64d2459 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java @@ -52,7 +52,7 @@ final class KllHeapItemsSketch extends KllItemsSketch { private boolean isLevelZeroSorted; private T minItem; private T maxItem; - private Object[] itemsArr; + private Object[] itemsArr; // null until first use, see ensureCapacity() /** * New instance heap constructor. @@ -77,7 +77,7 @@ final class KllHeapItemsSketch extends KllItemsSketch { isLevelZeroSorted = false; minItem = null; maxItem = null; - itemsArr = new Object[k]; + itemsArr = null; //allocated on first use, or by ensureCapacity() } /** @@ -119,12 +119,12 @@ final class KllHeapItemsSketch extends KllItemsSketch { n = segVal.n; minK = segVal.minK; isLevelZeroSorted = segVal.level0SortedFlag; - itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size final SketchStructure segStruct = segVal.sketchStructure; + //an empty image allocates no item storage; it is allocated on first use + itemsArr = (segStruct == COMPACT_EMPTY) ? null : new Object[levelsArr[segVal.numLevels]]; //updatable size if (segStruct == COMPACT_EMPTY) { minItem = null; maxItem = null; - itemsArr = new Object[k]; } else if (segStruct == COMPACT_SINGLE) { final int offset = N_LONG_ADR; final T item = serDe.deserializeFromMemorySegment(srcSeg, offset, 1)[0]; @@ -220,6 +220,7 @@ int getMinMaxSizeBytes() { T[] getRetainedItemsArray() { final int numRet = getNumRetained(); final T[] outArr = (T[]) Array.newInstance(serDe.getClassOfT(), numRet); + if (itemsArr == null) { return outArr; } //empty System.arraycopy(itemsArr, levelsArr[0], outArr, 0 , numRet); return outArr; } @@ -253,7 +254,7 @@ int getSingleItemSizeBytes() { @Override T[] getTotalItemsArray() { - if (n == 0) { return (T[]) Array.newInstance(serDe.getClassOfT(), k); } + if ((n == 0) || (itemsArr == null)) { return (T[]) Array.newInstance(serDe.getClassOfT(), k); } final T[] outArr = (T[]) Array.newInstance(serDe.getClassOfT(), itemsArr.length); System.arraycopy(itemsArr, 0, outArr, 0, itemsArr.length); return outArr; @@ -297,6 +298,22 @@ void setItemsArrayAt(final int index, final Object item) { itemsArr[index] = item; } + /** + * Allocates the item storage now, which a heap sketch otherwise allocates on first use. + * It has no effect if the storage already exists. + * @see KllSketch#ensureCapacity() + */ + @Override + public void ensureCapacity() { + if (itemsArr == null) { itemsArr = new Object[k]; } + } + + /** + * Returns true if the item storage has been allocated. + * @return true if the item storage has been allocated + */ + boolean hasItemStorage() { return itemsArr != null; } + @Override void setMaxItem(final Object item) { maxItem = (T) item; diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java index 21851b8b8..9eb0ff41a 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java @@ -52,7 +52,7 @@ final class KllHeapLongsSketch extends KllLongsSketch { private boolean isLevelZeroSorted; private long minLongItem; private long maxLongItem; - private long[] longItems; + private long[] longItems; // null until first use, see ensureCapacity() /** * New instance heap constructor with a given parameters k and m. @@ -76,7 +76,7 @@ final class KllHeapLongsSketch extends KllLongsSketch { isLevelZeroSorted = false; minLongItem = Long.MAX_VALUE; maxLongItem = Long.MIN_VALUE; - longItems = new long[k]; + longItems = null; //allocated on first use, or by ensureCapacity() } /** @@ -118,7 +118,7 @@ private KllHeapLongsSketch( if (segStructure == COMPACT_EMPTY) { minLongItem = Long.MAX_VALUE; maxLongItem = Long.MIN_VALUE; - longItems = new long[k]; + longItems = null; //allocated on first use } else if (segStructure == COMPACT_SINGLE) { final long item = srcSeg.get(JAVA_LONG_UNALIGNED, DATA_START_ADR_SINGLE_ITEM); @@ -221,7 +221,26 @@ byte[] getMinMaxByteArr() { //other restricted @Override - long[] getLongItemsArray() { return longItems; } + long[] getLongItemsArray() { + ensureCapacity(); //callers may write into the returned array + return longItems; + } + + /** + * Allocates the item storage now, which a heap sketch otherwise allocates on first use. + * It has no effect if the storage already exists. + * @see KllSketch#ensureCapacity() + */ + @Override + public void ensureCapacity() { + if (longItems == null) { longItems = new long[k]; } + } + + /** + * Returns true if the item storage has been allocated. + * @return true if the item storage has been allocated + */ + boolean hasItemStorage() { return longItems != null; } @Override long getLongSingleItem() { @@ -254,6 +273,7 @@ byte[] getRetainedItemsByteArr() { @Override byte[] getTotalItemsByteArr() { + if (longItems == null) { return new byte[k * Long.BYTES]; } //empty, all zeros final byte[] byteArr = new byte[longItems.length * Long.BYTES]; final MemorySegment wseg = MemorySegment.ofArray(byteArr); MemorySegment.copy(longItems, 0, wseg, JAVA_LONG_UNALIGNED, 0, longItems.length); @@ -303,6 +323,7 @@ void setNumLevels(final int numLevels) { @Override long[] getLongRetainedItemsArray() { + if (longItems == null) { return new long[0]; } return Arrays.copyOfRange(longItems, levelsArr[0], levelsArr[getNumLevels()]); } diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java index d71c75a85..5c147b2c7 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsSketch.java @@ -69,6 +69,7 @@ public abstract class KllItemsSketch extends KllSketch implements QuantilesGe * Create a new heap instance of this sketch with the default k = 200. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @param comparator to compare items * @param serDe Serializer / deserializer for an array of items, T[]. * @param The sketch data type. @@ -85,6 +86,7 @@ public static KllItemsSketch newHeapInstance( * k can be between DEFAULT_M and 65535, inclusive. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @param k parameter that controls size of the sketch and accuracy of estimates. * @param comparator to compare items * @param serDe Serializer / deserializer for items of type T and T[]. @@ -267,6 +269,7 @@ public final void merge(final KllSketch other) { if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } final KllItemsSketch othItmSk = (KllItemsSketch)other; if (othItmSk.isEmpty()) { return; } + ensureCapacity(); //first use allocates, before anything changes KllItemsHelper.mergeItemImpl(this, othItmSk, comparator); itemsSV = null; } @@ -282,7 +285,8 @@ public void reset() { setLevelsArray(new int[] {k, k}); setMinItem(null); setMaxItem(null); - setItemsArray(new Object[k]); + //a heap sketch releases its storage and allocates it again on first use + setItemsArray(null); itemsSV = null; } @@ -309,6 +313,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem public void update(final T item) { if (item == null) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + ensureCapacity(); //first use allocates, before anything changes KllItemsHelper.updateItem(this, item); itemsSV = null; } @@ -322,6 +327,7 @@ public void update(final T item, final long weight) { if (item == null) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + ensureCapacity(); //first use allocates, before anything changes if (weight == 1L) { KllItemsHelper.updateItem(this, item); } else { KllItemsHelper.updateItem(this, item, weight); } itemsSV = null; diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java index 4826d8392..22e6925a1 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -68,6 +68,7 @@ public abstract class KllLongsSketch extends KllSketch implements QuantilesLongs * Create a new heap instance of this sketch with the default k = 200. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @return new KllLongsSketch on the Java heap. */ public static KllLongsSketch newHeapInstance() { @@ -79,6 +80,7 @@ public static KllLongsSketch newHeapInstance() { * k can be between 8, inclusive, and 65535, inclusive. * The default k = 200 results in a normalized rank error of about * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}. * @param k parameter that controls size of the sketch and accuracy of estimates. * @return new KllLongsSketch on the Java heap. */ @@ -268,7 +270,7 @@ public double[] getRanks(final long[] quantiles, final QuantileSearchCriteria se @Override public QuantilesLongsSketchIterator iterator() { return new KllLongsSketchIterator( - getLongItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); + isEmpty() ? new long[0] : getLongItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); } @Override @@ -277,6 +279,7 @@ public final void merge(final KllSketch other) { if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } final KllLongsSketch otherLngSk = (KllLongsSketch)other; if (otherLngSk.isEmpty()) { return; } + ensureCapacity(); //first use allocates, before anything changes KllLongsHelper.mergeLongsImpl(this, otherLngSk); longsSV = null; } @@ -297,7 +300,8 @@ public final void reset() { setLevelsArray(new int[] {k, k}); setMinItem(Long.MAX_VALUE); setMaxItem(Long.MIN_VALUE); - setLongItemsArray(new long[k]); + //a heap sketch releases its storage and allocates it again on first use + setLongItemsArray(hasMemorySegment() ? new long[k] : null); longsSV = null; } @@ -323,6 +327,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem public void update(final long item) { // Align with KllDoublesSketch if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + ensureCapacity(); //first use allocates, before anything changes updateLong(this, item); longsSV = null; } @@ -369,6 +374,7 @@ public void update(final long item, final long weight) { // if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + ensureCapacity(); //first use allocates, before anything changes if (weight == 1L) { updateLong(this, item); } else if (weight < levelsArr[0]) { for (int i = 0; i < (int)weight; i++) { updateLong(this, item); } @@ -391,6 +397,7 @@ else if (weight < levelsArr[0]) { public void update(final long[] items, final int offset, final int length) { if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } if (length == 0) { return; } + ensureCapacity(); //first use allocates, before anything changes updateLong(items, offset, length); longsSV = null; } diff --git a/src/main/java/org/apache/datasketches/kll/KllSketch.java b/src/main/java/org/apache/datasketches/kll/KllSketch.java index 2a3f992dd..a91ee50e8 100644 --- a/src/main/java/org/apache/datasketches/kll/KllSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllSketch.java @@ -82,6 +82,13 @@ *

The default k of 200 yields a "single-sided" epsilon of about 1.33% and a * "double-sided" (PMF) epsilon of about 1.65%, with a confidence of 99%.

* + *

Memory allocation. A new heap sketch does not allocate storage for items until it is first used: + * the first update, or the first merge of a non-empty sketch. After that, the storage grows as the sketch + * needs it. An {@link OutOfMemoryError} from that first allocation therefore happens in the update or merge, + * not when the sketch is created. Call {@link #ensureCapacity()} right after creating the sketch to allocate + * the storage up front instead. A sketch backed by a MemorySegment is not affected. + * Results and the serialized form are the same either way.

+ * * @see KLL Sketch * @see QuantilesAPI * @@ -269,6 +276,16 @@ public final boolean isReadOnly() { */ public abstract void merge(KllSketch other); + /** + * Allocates the item storage of a heap sketch now, rather than on its first update or merge. + * The storage is the initial capacity of k items. Call this right after creating the sketch to move + * an {@link OutOfMemoryError} from the first update or merge to this call. It has no effect if the storage + * already exists, and none for a sketch backed by a MemorySegment. + * + *

{@link #reset()} releases the storage of a heap sketch, and it is allocated again on first use.

+ */ + public void ensureCapacity() { } + @Override public final String toString() { return toString(false, false); diff --git a/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java new file mode 100644 index 000000000..e54275bd1 --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.testng.annotations.Test; + +/** + * Checks that heap KLL sketches allocate item storage on first use, or when ensureCapacity() is called. + * KLL compaction is random, so images are compared byte for byte only below k items, before any compaction. + */ +public class KllLazyAllocationTest { + private static final int K = 200; + private static final ArrayOfStringsSerDe SERDE = new ArrayOfStringsSerDe(); + + private static KllHeapDoublesSketch heapDoubles(final KllDoublesSketch sk) { return (KllHeapDoublesSketch) sk; } + + @Test + public void doublesEmptySketchHoldsNoStorage() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + assertFalse(heapDoubles(sk).hasItemStorage()); + sk.update(Double.NaN); //ignored, so not a first use + assertTrue(sk.isEmpty()); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumRetained(), 0); + assertFalse(sk.iterator().next()); + sk.toString(true, true); + + final KllDoublesSketch ref = KllDoublesSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + + sk.merge(KllDoublesSketch.newHeapInstance(K)); //merging an empty sketch is not a first use + assertFalse(heapDoubles(sk).hasItemStorage()); + + final KllDoublesSketch copy = KllDoublesSketch.heapify(MemorySegment.ofArray(sk.toByteArray())); + assertFalse(heapDoubles(copy).hasItemStorage()); + assertTrue(copy.isEmpty()); + assertFalse(heapDoubles(sk).hasItemStorage()); + assertFalse(heapDoubles(copy).hasItemStorage()); + } + + @Test + public void doublesEnsureCapacity() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + sk.ensureCapacity(); + final KllHeapDoublesSketch heap = heapDoubles(sk); + assertTrue(heap.hasItemStorage()); + final double[] storage = heap.getDoubleItemsArray(); + assertEquals(storage.length, K); + sk.ensureCapacity(); //no effect once allocated + assertSame(heap.getDoubleItemsArray(), storage); + + final KllDoublesSketch lazy = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < K; i++) { sk.update(i); lazy.update(i); } //fills the initial storage, no compaction + assertSame(heap.getDoubleItemsArray(), storage); //nothing more was allocated + assertEquals(sk.toByteArray(), lazy.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(lazy, true)); + + final double[] before = heap.getDoubleItemsArray(); + final byte[] image = sk.toByteArray(); + sk.ensureCapacity(); //no effect after updates + assertSame(heap.getDoubleItemsArray(), before); + assertEquals(sk.toByteArray(), image); + } + + @Test + public void doublesFirstUseAllocates() { + final KllDoublesSketch single = KllDoublesSketch.newHeapInstance(K); + single.update(1.0); + assertTrue(heapDoubles(single).hasItemStorage()); + assertEquals(single.getMinItem(), 1.0); + + final KllDoublesSketch weighted = KllDoublesSketch.newHeapInstance(K); + weighted.update(2.0, 3); + assertTrue(heapDoubles(weighted).hasItemStorage()); + assertEquals(weighted.getN(), 3); + + final KllDoublesSketch heavy = KllDoublesSketch.newHeapInstance(K); + heavy.update(2.0, 10_000); //takes the merge path + assertEquals(heavy.getN(), 10_000); + assertEquals(heavy.getRank(2.0, INCLUSIVE), 1.0); + + final KllDoublesSketch vector = KllDoublesSketch.newHeapInstance(K); + vector.update(new double[] {3.0, Double.NaN, 4.0}, 0, 3); + assertEquals(vector.getN(), 2); + assertEquals(vector.getMaxItem(), 4.0); + + final KllDoublesSketch merged = KllDoublesSketch.newHeapInstance(K); + final KllDoublesSketch other = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 10_000; i++) { other.update(i); } + merged.merge(other); + assertTrue(heapDoubles(merged).hasItemStorage()); + assertEquals(merged.getN(), 10_000); + assertEquals(merged.getMinItem(), 0.0); + assertEquals(merged.getMaxItem(), 9999.0); + final double r = merged.getRank(5000, INCLUSIVE); + assertTrue(Math.abs(r - 0.5) < 0.05, "rank " + r); + + final KllDoublesSketch fromSingle = KllDoublesSketch.newHeapInstance(K); + fromSingle.merge(KllDoublesSketch.heapify(MemorySegment.ofArray(single.toByteArray()))); + assertEquals(fromSingle.toByteArray(), single.toByteArray()); + } + + @Test + public void doublesResetReleasesStorage() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 1000; i++) { sk.update(i); } + sk.reset(); + assertFalse(heapDoubles(sk).hasItemStorage()); + assertTrue(sk.isEmpty()); + final KllDoublesSketch fresh = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 100; i++) { sk.update(i); fresh.update(i); } + assertEquals(sk.toByteArray(), fresh.toByteArray()); + } + + @Test + public void directSketchIsUnaffected() { + final int bytes = KllSketch.getMaxSerializedSizeBytes(K, 1000, KllSketch.SketchType.KLL_DOUBLES_SKETCH, true); + final KllDoublesSketch sk = KllDoublesSketch.newDirectInstance(K, MemorySegment.ofArray(new byte[bytes]), null); + sk.ensureCapacity(); //no effect + for (int i = 0; i < 1000; i++) { sk.update(i); } + sk.reset(); + assertTrue(sk.isEmpty()); + sk.update(1.0); + assertEquals(sk.getN(), 1); + } + + @Test + public void floatsLazy() { + final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(K); + final KllHeapFloatsSketch heap = (KllHeapFloatsSketch) sk; + assertFalse(heap.hasItemStorage()); + sk.update(Float.NaN); + assertFalse(heap.hasItemStorage()); + final KllFloatsSketch ref = KllFloatsSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertEquals(heap.getFloatItemsArray().length, K); //reading the array for writing allocates it + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + + final KllFloatsSketch lazy = KllFloatsSketch.newHeapInstance(K); + assertFalse(((KllHeapFloatsSketch) lazy).hasItemStorage()); + for (int i = 0; i < K; i++) { ref.update(i); lazy.update(i); } + assertEquals(ref.toByteArray(), lazy.toByteArray()); + lazy.reset(); + assertFalse(((KllHeapFloatsSketch) lazy).hasItemStorage()); + final KllFloatsSketch merged = KllFloatsSketch.newHeapInstance(K); + merged.merge(ref); + assertEquals(merged.getN(), K); + } + + @Test + public void longsLazy() { + final KllLongsSketch sk = KllLongsSketch.newHeapInstance(K); + final KllHeapLongsSketch heap = (KllHeapLongsSketch) sk; + assertFalse(heap.hasItemStorage()); + final KllLongsSketch ref = KllLongsSketch.newHeapInstance(K); + ref.ensureCapacity(); + assertTrue(((KllHeapLongsSketch) ref).hasItemStorage()); + assertEquals(sk.toByteArray(), ref.toByteArray()); + assertEquals(KllHelper.toByteArray(sk, true), KllHelper.toByteArray(ref, true)); + assertFalse(heap.hasItemStorage()); + + for (int i = 0; i < K; i++) { sk.update(i); ref.update(i); } + assertEquals(sk.toByteArray(), ref.toByteArray()); + sk.reset(); + assertFalse(heap.hasItemStorage()); + final KllLongsSketch merged = KllLongsSketch.newHeapInstance(K); + merged.merge(ref); + assertEquals(merged.getN(), K); + assertEquals(merged.getMaxItem(), K - 1); + } + + @Test + public void itemsLazy() { + final KllItemsSketch sk = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + final KllHeapItemsSketch heap = (KllHeapItemsSketch) sk; + assertFalse(heap.hasItemStorage()); + sk.update(null); //ignored + assertFalse(heap.hasItemStorage()); + assertFalse(sk.iterator().next()); + sk.toString(true, true); + assertEquals(heap.getRetainedItemsArray().length, 0); + + final KllItemsSketch ref = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + ref.ensureCapacity(); + assertTrue(((KllHeapItemsSketch) ref).hasItemStorage()); + assertEquals(sk.toByteArray(), ref.toByteArray()); + + sk.merge(KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE)); + assertFalse(heap.hasItemStorage()); + final KllItemsSketch copy = KllItemsSketch.heapify(MemorySegment.ofArray(sk.toByteArray()), + Comparator.naturalOrder(), SERDE); + assertFalse(((KllHeapItemsSketch) copy).hasItemStorage()); + + for (int i = 0; i < K; i++) { sk.update(Integer.toString(i)); ref.update(Integer.toString(i)); } + assertTrue(heap.hasItemStorage()); + assertEquals(sk.toByteArray(), ref.toByteArray()); + + final KllItemsSketch weighted = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + weighted.update("a", 10_000); + assertEquals(weighted.getN(), 10_000); + + final KllItemsSketch merged = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + merged.merge(ref); + assertEquals(merged.getN(), K); + //merging copies level 0 one item at a time, which reverses its order, so compare contents, not bytes + assertEquals(merged.getMinItem(), ref.getMinItem()); + assertEquals(merged.getMaxItem(), ref.getMaxItem()); + assertEquals(merged.getRank("5", INCLUSIVE), ref.getRank("5", INCLUSIVE)); + + sk.reset(); + assertFalse(heap.hasItemStorage()); + assertTrue(sk.isEmpty()); + sk.update("z"); + assertEquals(sk.getMinItem(), "z"); + } +} From 02a7947428e99eb3262093af7e0ed90f3719f515 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 16 Nov 2026 14:02:58 -0800 Subject: [PATCH 2/2] KLL: heapify allocates item storage only when the image holds items An updatable image of an empty sketch (no items, base level only) now heapifies without item storage, like a compact empty image; it is allocated on first use. The items sketch now allocates its array only in the single-item and full branches, so an unsupported updatable image is rejected without allocating first. Co-Authored-By: Claude Opus 5.5 --- .../kll/KllHeapDoublesSketch.java | 10 ++- .../datasketches/kll/KllHeapFloatsSketch.java | 10 ++- .../datasketches/kll/KllHeapItemsSketch.java | 5 +- .../datasketches/kll/KllHeapLongsSketch.java | 10 ++- .../apache/datasketches/kll/KllHelper.java | 12 ++++ .../kll/KllLazyAllocationTest.java | 64 +++++++++++++++++++ 6 files changed, 100 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java index dc82e0796..59e391b47 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapDoublesSketch.java @@ -146,9 +146,13 @@ else if (segStructure == COMPACT_FULL) { offsetBytes += Double.BYTES; maxDoubleItem = srcSeg.get(JAVA_DOUBLE_UNALIGNED, offsetBytes); offsetBytes += Double.BYTES; - final int capacityItems = levelsArr[getNumLevels()]; - doubleItems = new double[capacityItems]; - MemorySegment.copy(srcSeg, JAVA_DOUBLE_UNALIGNED, offsetBytes, doubleItems, 0, capacityItems); + if (KllHelper.isEmptyBaseLevels(n, k, levelsArr)) { + doubleItems = null; //allocated on first use + } else { + final int capacityItems = levelsArr[getNumLevels()]; + doubleItems = new double[capacityItems]; + MemorySegment.copy(srcSeg, JAVA_DOUBLE_UNALIGNED, offsetBytes, doubleItems, 0, capacityItems); + } } } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java index becba9a40..9704f9887 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java @@ -146,9 +146,13 @@ else if (segStructure == COMPACT_FULL) { offsetBytes += Float.BYTES; maxFloatItem = srcSeg.get(JAVA_FLOAT_UNALIGNED, offsetBytes); offsetBytes += Float.BYTES; - final int capacityItems = levelsArr[getNumLevels()]; - floatItems = new float[capacityItems]; - MemorySegment.copy(srcSeg, JAVA_FLOAT_UNALIGNED, offsetBytes, floatItems, 0, capacityItems); + if (KllHelper.isEmptyBaseLevels(n, k, levelsArr)) { + floatItems = null; //allocated on first use + } else { + final int capacityItems = levelsArr[getNumLevels()]; + floatItems = new float[capacityItems]; + MemorySegment.copy(srcSeg, JAVA_FLOAT_UNALIGNED, offsetBytes, floatItems, 0, capacityItems); + } } } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java index 9d64d2459..9944a6fe7 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java @@ -120,16 +120,16 @@ final class KllHeapItemsSketch extends KllItemsSketch { minK = segVal.minK; isLevelZeroSorted = segVal.level0SortedFlag; final SketchStructure segStruct = segVal.sketchStructure; - //an empty image allocates no item storage; it is allocated on first use - itemsArr = (segStruct == COMPACT_EMPTY) ? null : new Object[levelsArr[segVal.numLevels]]; //updatable size if (segStruct == COMPACT_EMPTY) { minItem = null; maxItem = null; + itemsArr = null; //allocated on first use } else if (segStruct == COMPACT_SINGLE) { final int offset = N_LONG_ADR; final T item = serDe.deserializeFromMemorySegment(srcSeg, offset, 1)[0]; minItem = item; maxItem = item; + itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size itemsArr[k - 1] = item; } else if (segStruct == COMPACT_FULL) { int offset = DATA_START_ADR + (segVal.numLevels * Integer.BYTES); @@ -139,6 +139,7 @@ final class KllHeapItemsSketch extends KllItemsSketch { offset += serDe.sizeOf(maxItem); final int numRetained = levelsArr[segVal.numLevels] - levelsArr[0]; final Object[] retItems = serDe.deserializeFromMemorySegment(srcSeg, offset, numRetained); + itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size System.arraycopy(retItems, 0, itemsArr, levelsArr[0], numRetained); } else { //segStruct == UPDATABLE throw new SketchesArgumentException(UNSUPPORTED_MSG + "UPDATABLE"); diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java index 9eb0ff41a..864399808 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapLongsSketch.java @@ -146,9 +146,13 @@ else if (segStructure == COMPACT_FULL) { offsetBytes += Long.BYTES; maxLongItem = srcSeg.get(JAVA_LONG_UNALIGNED, offsetBytes); offsetBytes += Long.BYTES; - final int capacityItems = levelsArr[getNumLevels()]; - longItems = new long[capacityItems]; - MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, offsetBytes, longItems, 0, capacityItems); + if (KllHelper.isEmptyBaseLevels(n, k, levelsArr)) { + longItems = null; //allocated on first use + } else { + final int capacityItems = levelsArr[getNumLevels()]; + longItems = new long[capacityItems]; + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, offsetBytes, longItems, 0, capacityItems); + } } } diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index 4b6256cde..36492ad87 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -101,6 +101,18 @@ static class LevelStats { 847288609443L, 2541865828329L, 7625597484987L, 22876792454961L, 68630377364883L, 205891132094649L}; + /** + * Returns true if an updatable image holds no items and only the base level, as an empty + * sketch does. Heapifying such an image allocates no item storage; it is allocated on first use. + * @param n the number of items the image has seen + * @param k the k of the image + * @param levelsArr the full levels array of the image + * @return true if the image is an empty updatable image with only the base level + */ + static boolean isEmptyBaseLevels(final long n, final int k, final int[] levelsArr) { + return (n == 0) && (levelsArr.length == 2) && (levelsArr[0] == k) && (levelsArr[1] == k); + } + /** * Checks the validity of the given k * @param k must be greater than 7 and less than 65536. diff --git a/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java index e54275bd1..875d6cc7c 100644 --- a/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllLazyAllocationTest.java @@ -28,6 +28,7 @@ import java.lang.foreign.MemorySegment; import java.util.Comparator; +import java.util.List; import org.apache.datasketches.common.ArrayOfStringsSerDe; import org.testng.annotations.Test; @@ -244,4 +245,67 @@ public void itemsLazy() { sk.update("z"); assertEquals(sk.getMinItem(), "z"); } + + //An updatable image of an empty sketch heapifies without item storage, like a compact empty image. + @Test + public void updatableEmptyImageHoldsNoStorage() { + final KllDoublesSketch grown = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < (10 * K); i++) { grown.update(i); } + grown.reset(); //back to the base level + final KllDoublesSketch[] empties = { KllDoublesSketch.newHeapInstance(K), grown }; + for (final KllDoublesSketch empty : empties) { + final byte[] image = KllHelper.toByteArray(empty, true); + final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(image)); + assertFalse(heapDoubles(sk).hasItemStorage()); + assertTrue(sk.isEmpty()); + assertEquals(KllHelper.toByteArray(sk, true), image); + assertEquals(sk.toByteArray(), empty.toByteArray()); + sk.update(1.0); + assertTrue(heapDoubles(sk).hasItemStorage()); + assertEquals(sk.getN(), 1); + } + + final byte[] floatsImage = KllHelper.toByteArray(KllFloatsSketch.newHeapInstance(K), true); + final KllFloatsSketch floats = KllFloatsSketch.heapify(MemorySegment.ofArray(floatsImage)); + assertFalse(((KllHeapFloatsSketch) floats).hasItemStorage()); + assertEquals(KllHelper.toByteArray(floats, true), floatsImage); + floats.update(1.0f); + assertTrue(((KllHeapFloatsSketch) floats).hasItemStorage()); + + final byte[] longsImage = KllHelper.toByteArray(KllLongsSketch.newHeapInstance(K), true); + final KllLongsSketch longs = KllLongsSketch.heapify(MemorySegment.ofArray(longsImage)); + assertFalse(((KllHeapLongsSketch) longs).hasItemStorage()); + assertEquals(KllHelper.toByteArray(longs, true), longsImage); + longs.update(1L); + assertTrue(((KllHeapLongsSketch) longs).hasItemStorage()); + } + + //Updatable images that hold items still heapify with their item storage. + @Test + public void updatableImageWithItemsHasStorage() { + for (final int n : new int[] {1, K - 1, 10 * K}) { + final KllDoublesSketch src = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < n; i++) { src.update(i); } + final byte[] image = KllHelper.toByteArray(src, true); + final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(image)); + assertTrue(heapDoubles(sk).hasItemStorage()); + assertEquals(KllHelper.toByteArray(sk, true), image); + assertEquals(sk.toByteArray(), src.toByteArray()); + } + } + + //A compact items image allocates item storage only for a single item or a full sketch. + @Test + public void itemsHeapifyAllocatesOnlyWithItems() { + final KllItemsSketch single = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + single.update("a"); + final KllItemsSketch full = KllItemsSketch.newHeapInstance(K, Comparator.naturalOrder(), SERDE); + for (int i = 0; i < (10 * K); i++) { full.update(Integer.toString(i)); } + for (final KllItemsSketch src : List.of(single, full)) { + final KllItemsSketch copy = KllItemsSketch.heapify(MemorySegment.ofArray(src.toByteArray()), + Comparator.naturalOrder(), SERDE); + assertTrue(((KllHeapItemsSketch) copy).hasItemStorage()); + assertEquals(copy.toByteArray(), src.toByteArray()); + } + } }