Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ public abstract class KllDoublesSketch extends KllSketch implements QuantilesDou
* Create a new heap instance of this sketch with the default <em>k = 200</em>.
* The default <em>k</em> = 200 results in a normalized rank error of about
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
* The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}.
* @return new KllDoublesSketch on the Java heap.
*/
public static KllDoublesSketch newHeapInstance() {
Expand All @@ -79,6 +80,7 @@ public static KllDoublesSketch newHeapInstance() {
* <em>k</em> can be between 8, inclusive, and 65535, inclusive.
* The default <em>k</em> = 200 results in a normalized rank error of about
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
* The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}.
* @param k parameter that controls size of the sketch and accuracy of estimates.
* @return new KllDoublesSketch on the Java heap.
*/
Expand Down Expand Up @@ -268,7 +270,7 @@ public double[] getRanks(final double[] quantiles, final QuantileSearchCriteria
@Override
public QuantilesDoublesSketchIteratorAPI iterator() {
return new KllDoublesSketchIterator(
getDoubleItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels());
isEmpty() ? new double[0] : getDoubleItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels());
}

@Override
Expand All @@ -277,6 +279,7 @@ public final void merge(final KllSketch other) {
if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); }
final KllDoublesSketch othDblSk = (KllDoublesSketch)other;
if (othDblSk.isEmpty()) { return; }
ensureCapacity(); //first use allocates, before anything changes
KllDoublesHelper.mergeDoubleImpl(this, othDblSk);
doublesSV = null;
}
Expand All @@ -297,7 +300,8 @@ public final void reset() {
setLevelsArray(new int[] {k, k});
setMinItem(Double.NaN);
setMaxItem(Double.NaN);
setDoubleItemsArray(new double[k]);
//a heap sketch releases its storage and allocates it again on first use
setDoubleItemsArray(hasMemorySegment() ? new double[k] : null);
doublesSV = null;
}

Expand All @@ -323,6 +327,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem
public void update(final double item) {
if (Double.isNaN(item)) { return; } //ignore
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
ensureCapacity(); //first use allocates, before anything changes
updateDouble(this, item);
doublesSV = null;
}
Expand Down Expand Up @@ -369,6 +374,7 @@ public void update(final double item, final long weight) {
if (Double.isNaN(item)) { return; } //ignore
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); }
ensureCapacity(); //first use allocates, before anything changes
if (weight == 1L) { updateDouble(this, item); }
else if (weight < levelsArr[0]) {
for (int i = 0; i < (int)weight; i++) { updateDouble(this, item); }
Expand All @@ -393,6 +399,7 @@ else if (weight < levelsArr[0]) {
public void update(final double[] items, final int offset, final int length) {
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
if (length == 0) { return; }
ensureCapacity(); //first use allocates, before anything changes
if (!hasNaN(items, offset, length)) {
updateDouble(items, offset, length); //fast path
doublesSV = null;
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ public abstract class KllFloatsSketch extends KllSketch implements QuantilesFloa
* Create a new heap instance of this sketch with the default <em>k = 200</em>.
* The default <em>k</em> = 200 results in a normalized rank error of about
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
* The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}.
* @return new KllFloatsSketch on the Java heap.
*/
public static KllFloatsSketch newHeapInstance() {
Expand All @@ -79,6 +80,7 @@ public static KllFloatsSketch newHeapInstance() {
* <em>k</em> can be between 8, inclusive, and 65535, inclusive.
* The default <em>k</em> = 200 results in a normalized rank error of about
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
* The sketch allocates its item storage on first use; see {@link KllSketch#ensureCapacity()}.
* @param k parameter that controls size of the sketch and accuracy of estimates.
* @return new KllFloatsSketch on the Java heap.
*/
Expand Down Expand Up @@ -268,7 +270,7 @@ public double[] getRanks(final float[] quantiles, final QuantileSearchCriteria s
@Override
public QuantilesFloatsSketchIterator iterator() {
return new KllFloatsSketchIterator(
getFloatItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels());
isEmpty() ? new float[0] : getFloatItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels());
}

@Override
Expand All @@ -277,6 +279,7 @@ public final void merge(final KllSketch other) {
if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); }
final KllFloatsSketch othFltSk = (KllFloatsSketch)other;
if (othFltSk.isEmpty()) { return; }
ensureCapacity(); //first use allocates, before anything changes
KllFloatsHelper.mergeFloatImpl(this, othFltSk);
floatsSV = null;
}
Expand All @@ -297,7 +300,8 @@ public final void reset() {
setLevelsArray(new int[] {k, k});
setMinItem(Float.NaN);
setMaxItem(Float.NaN);
setFloatItemsArray(new float[k]);
//a heap sketch releases its storage and allocates it again on first use
setFloatItemsArray(hasMemorySegment() ? new float[k] : null);
floatsSV = null;
}

Expand All @@ -323,6 +327,7 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem
public void update(final float item) {
if (Float.isNaN(item)) { return; } //ignore
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
ensureCapacity(); //first use allocates, before anything changes
updateFloat(this, item);
floatsSV = null;
}
Expand Down Expand Up @@ -369,6 +374,7 @@ public void update(final float item, final long weight) {
if (Float.isNaN(item)) { return; } //ignore
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); }
ensureCapacity(); //first use allocates, before anything changes
if (weight == 1L) { updateFloat(this, item); }
else if (weight < levelsArr[0]) {
for (int i = 0; i < (int)weight; i++) { updateFloat(this, item); }
Expand All @@ -393,6 +399,7 @@ else if (weight < levelsArr[0]) {
public void update(final float[] items, final int offset, final int length) {
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
if (length == 0) { return; }
ensureCapacity(); //first use allocates, before anything changes
if (!hasNaN(items, offset, length)) {
updateFloat(items, offset, length); //fast path
floatsSV = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ final class KllHeapDoublesSketch extends KllDoublesSketch {
private boolean isLevelZeroSorted;
private double minDoubleItem;
private double maxDoubleItem;
private double[] doubleItems;
private double[] doubleItems; // null until first use, see ensureCapacity()

/**
* New instance heap constructor with a given parameters <em>k</em> and <em>m</em>.
Expand All @@ -76,7 +76,7 @@ final class KllHeapDoublesSketch extends KllDoublesSketch {
isLevelZeroSorted = false;
minDoubleItem = Double.NaN;
maxDoubleItem = Double.NaN;
doubleItems = new double[k];
doubleItems = null; //allocated on first use, or by ensureCapacity()
}

/**
Expand Down Expand Up @@ -118,7 +118,7 @@ private KllHeapDoublesSketch(
if (segStructure == COMPACT_EMPTY) {
minDoubleItem = Double.NaN;
maxDoubleItem = Double.NaN;
doubleItems = new double[k];
doubleItems = null; //allocated on first use
}
else if (segStructure == COMPACT_SINGLE) {
final double item = srcSeg.get(JAVA_DOUBLE_UNALIGNED, DATA_START_ADR_SINGLE_ITEM);
Expand Down Expand Up @@ -146,9 +146,13 @@ else if (segStructure == COMPACT_FULL) {
offsetBytes += Double.BYTES;
maxDoubleItem = srcSeg.get(JAVA_DOUBLE_UNALIGNED, offsetBytes);
offsetBytes += Double.BYTES;
final int capacityItems = levelsArr[getNumLevels()];
doubleItems = new double[capacityItems];
MemorySegment.copy(srcSeg, JAVA_DOUBLE_UNALIGNED, offsetBytes, doubleItems, 0, capacityItems);
if (KllHelper.isEmptyBaseLevels(n, k, levelsArr)) {
doubleItems = null; //allocated on first use
} else {
final int capacityItems = levelsArr[getNumLevels()];
doubleItems = new double[capacityItems];
MemorySegment.copy(srcSeg, JAVA_DOUBLE_UNALIGNED, offsetBytes, doubleItems, 0, capacityItems);
}
}
}

Expand Down Expand Up @@ -221,7 +225,26 @@ byte[] getMinMaxByteArr() {
//other restricted

@Override
double[] getDoubleItemsArray() { return doubleItems; }
double[] getDoubleItemsArray() {
ensureCapacity(); //callers may write into the returned array
return doubleItems;
}

/**
* Allocates the item storage now, which a heap sketch otherwise allocates on first use.
* It has no effect if the storage already exists.
* @see KllSketch#ensureCapacity()
*/
@Override
public void ensureCapacity() {
if (doubleItems == null) { doubleItems = new double[k]; }
}

/**
* Returns true if the item storage has been allocated.
* @return true if the item storage has been allocated
*/
boolean hasItemStorage() { return doubleItems != null; }

@Override
double getDoubleSingleItem() {
Expand Down Expand Up @@ -254,6 +277,7 @@ byte[] getRetainedItemsByteArr() {

@Override
byte[] getTotalItemsByteArr() {
if (doubleItems == null) { return new byte[k * Double.BYTES]; } //empty, all zeros
final byte[] byteArr = new byte[doubleItems.length * Double.BYTES];
final MemorySegment wseg = MemorySegment.ofArray(byteArr);
MemorySegment.copy(doubleItems, 0, wseg, JAVA_DOUBLE_UNALIGNED, 0, doubleItems.length);
Expand Down Expand Up @@ -303,6 +327,7 @@ void setNumLevels(final int numLevels) {

@Override
double[] getDoubleRetainedItemsArray() {
if (doubleItems == null) { return new double[0]; }
return Arrays.copyOfRange(doubleItems, levelsArr[0], levelsArr[getNumLevels()]);
}

Expand Down
39 changes: 32 additions & 7 deletions src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ final class KllHeapFloatsSketch extends KllFloatsSketch {
private boolean isLevelZeroSorted;
private float minFloatItem;
private float maxFloatItem;
private float[] floatItems;
private float[] floatItems; // null until first use, see ensureCapacity()

/**
* New instance heap constructor with a given parameters <em>k</em> and <em>m</em>.
Expand All @@ -76,7 +76,7 @@ final class KllHeapFloatsSketch extends KllFloatsSketch {
isLevelZeroSorted = false;
minFloatItem = Float.NaN;
maxFloatItem = Float.NaN;
floatItems = new float[k];
floatItems = null; //allocated on first use, or by ensureCapacity()
}

/**
Expand Down Expand Up @@ -118,7 +118,7 @@ private KllHeapFloatsSketch(
if (segStructure == COMPACT_EMPTY) {
minFloatItem = Float.NaN;
maxFloatItem = Float.NaN;
floatItems = new float[k];
floatItems = null; //allocated on first use
}
else if (segStructure == COMPACT_SINGLE) {
final float item = srcSeg.get(JAVA_FLOAT_UNALIGNED, DATA_START_ADR_SINGLE_ITEM);
Expand Down Expand Up @@ -146,9 +146,13 @@ else if (segStructure == COMPACT_FULL) {
offsetBytes += Float.BYTES;
maxFloatItem = srcSeg.get(JAVA_FLOAT_UNALIGNED, offsetBytes);
offsetBytes += Float.BYTES;
final int capacityItems = levelsArr[getNumLevels()];
floatItems = new float[capacityItems];
MemorySegment.copy(srcSeg, JAVA_FLOAT_UNALIGNED, offsetBytes, floatItems, 0, capacityItems);
if (KllHelper.isEmptyBaseLevels(n, k, levelsArr)) {
floatItems = null; //allocated on first use
} else {
final int capacityItems = levelsArr[getNumLevels()];
floatItems = new float[capacityItems];
MemorySegment.copy(srcSeg, JAVA_FLOAT_UNALIGNED, offsetBytes, floatItems, 0, capacityItems);
}
}
}

Expand Down Expand Up @@ -221,7 +225,26 @@ byte[] getMinMaxByteArr() {
//other restricted

@Override
float[] getFloatItemsArray() { return floatItems; }
float[] getFloatItemsArray() {
ensureCapacity(); //callers may write into the returned array
return floatItems;
}

/**
* Allocates the item storage now, which a heap sketch otherwise allocates on first use.
* It has no effect if the storage already exists.
* @see KllSketch#ensureCapacity()
*/
@Override
public void ensureCapacity() {
if (floatItems == null) { floatItems = new float[k]; }
}

/**
* Returns true if the item storage has been allocated.
* @return true if the item storage has been allocated
*/
boolean hasItemStorage() { return floatItems != null; }

@Override
float getFloatSingleItem() {
Expand Down Expand Up @@ -254,6 +277,7 @@ byte[] getRetainedItemsByteArr() {

@Override
byte[] getTotalItemsByteArr() {
if (floatItems == null) { return new byte[k * Float.BYTES]; } //empty, all zeros
final byte[] byteArr = new byte[floatItems.length * Float.BYTES];
final MemorySegment wseg = MemorySegment.ofArray(byteArr);
MemorySegment.copy(floatItems, 0, wseg, JAVA_FLOAT_UNALIGNED, 0, floatItems.length);
Expand Down Expand Up @@ -303,6 +327,7 @@ void setNumLevels(final int numLevels) {

@Override
float[] getFloatRetainedItemsArray() {
if (floatItems == null) { return new float[0]; }
return Arrays.copyOfRange(floatItems, levelsArr[0], levelsArr[getNumLevels()]);
}

Expand Down
28 changes: 23 additions & 5 deletions src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ final class KllHeapItemsSketch<T> extends KllItemsSketch<T> {
private boolean isLevelZeroSorted;
private T minItem;
private T maxItem;
private Object[] itemsArr;
private Object[] itemsArr; // null until first use, see ensureCapacity()

/**
* New instance heap constructor.
Expand All @@ -77,7 +77,7 @@ final class KllHeapItemsSketch<T> extends KllItemsSketch<T> {
isLevelZeroSorted = false;
minItem = null;
maxItem = null;
itemsArr = new Object[k];
itemsArr = null; //allocated on first use, or by ensureCapacity()
}

/**
Expand Down Expand Up @@ -119,17 +119,17 @@ final class KllHeapItemsSketch<T> extends KllItemsSketch<T> {
n = segVal.n;
minK = segVal.minK;
isLevelZeroSorted = segVal.level0SortedFlag;
itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size
final SketchStructure segStruct = segVal.sketchStructure;
if (segStruct == COMPACT_EMPTY) {
minItem = null;
maxItem = null;
itemsArr = new Object[k];
itemsArr = null; //allocated on first use
} else if (segStruct == COMPACT_SINGLE) {
final int offset = N_LONG_ADR;
final T item = serDe.deserializeFromMemorySegment(srcSeg, offset, 1)[0];
minItem = item;
maxItem = item;
itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size
itemsArr[k - 1] = item;
} else if (segStruct == COMPACT_FULL) {
int offset = DATA_START_ADR + (segVal.numLevels * Integer.BYTES);
Expand All @@ -139,6 +139,7 @@ final class KllHeapItemsSketch<T> extends KllItemsSketch<T> {
offset += serDe.sizeOf(maxItem);
final int numRetained = levelsArr[segVal.numLevels] - levelsArr[0];
final Object[] retItems = serDe.deserializeFromMemorySegment(srcSeg, offset, numRetained);
itemsArr = new Object[levelsArr[segVal.numLevels]]; //updatable size
System.arraycopy(retItems, 0, itemsArr, levelsArr[0], numRetained);
} else { //segStruct == UPDATABLE
throw new SketchesArgumentException(UNSUPPORTED_MSG + "UPDATABLE");
Expand Down Expand Up @@ -220,6 +221,7 @@ int getMinMaxSizeBytes() {
T[] getRetainedItemsArray() {
final int numRet = getNumRetained();
final T[] outArr = (T[]) Array.newInstance(serDe.getClassOfT(), numRet);
if (itemsArr == null) { return outArr; } //empty
System.arraycopy(itemsArr, levelsArr[0], outArr, 0 , numRet);
return outArr;
}
Expand Down Expand Up @@ -253,7 +255,7 @@ int getSingleItemSizeBytes() {

@Override
T[] getTotalItemsArray() {
if (n == 0) { return (T[]) Array.newInstance(serDe.getClassOfT(), k); }
if ((n == 0) || (itemsArr == null)) { return (T[]) Array.newInstance(serDe.getClassOfT(), k); }
final T[] outArr = (T[]) Array.newInstance(serDe.getClassOfT(), itemsArr.length);
System.arraycopy(itemsArr, 0, outArr, 0, itemsArr.length);
return outArr;
Expand Down Expand Up @@ -297,6 +299,22 @@ void setItemsArrayAt(final int index, final Object item) {
itemsArr[index] = item;
}

/**
* Allocates the item storage now, which a heap sketch otherwise allocates on first use.
* It has no effect if the storage already exists.
* @see KllSketch#ensureCapacity()
*/
@Override
public void ensureCapacity() {
if (itemsArr == null) { itemsArr = new Object[k]; }
}

/**
* Returns true if the item storage has been allocated.
* @return true if the item storage has been allocated
*/
boolean hasItemStorage() { return itemsArr != null; }

@Override
void setMaxItem(final Object item) {
maxItem = (T) item;
Expand Down
Loading
Loading