Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}

@Override
void incNumLevels() {
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
int numLevels = getMemorySegmentNumLevels(wseg);
setMemorySegmentNumLevels(wseg, ++numLevels);
}

@Override
boolean isLevelZeroSorted() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}

@Override
void incNumLevels() {
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
int numLevels = getMemorySegmentNumLevels(wseg);
setMemorySegmentNumLevels(wseg, ++numLevels);
}

@Override
boolean isLevelZeroSorted() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -317,12 +317,6 @@ void incN(final int increment) {
setMemorySegmentN(wseg, getMemorySegmentN(wseg) + increment);
}

@Override
void incNumLevels() {
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
int numLevels = getMemorySegmentNumLevels(wseg);
setMemorySegmentNumLevels(wseg, ++numLevels);
}

@Override
boolean isLevelZeroSorted() {
Expand Down
17 changes: 10 additions & 7 deletions src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger

// THE NEW ITEMS ARRAY
myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
? myCurDoubleItemsArr
: new double[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];

//calculate the new levels array length
Expand All @@ -204,6 +197,16 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}

// THE NEW ITEMS ARRAY
//the levels array is allocated first, because this may be the current items array,
//and once it is overwritten nothing else may fail
myNewDoubleItemsArr = (targetItemCount == myCurDoubleItemsArr.length)
? myCurDoubleItemsArr
: new double[targetItemCount];

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewDoubleItemsArr, freeSpaceAtBottom, curItemCount);

//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,14 +334,15 @@ public void update(final double item) {

//Also Called from KllDoublesHelper::merge
static void updateDouble(final KllDoublesSketch dblSk, final double item) {
dblSk.updateMinMax(item);
int freeSpace = dblSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
KllDoublesHelper.compressWhileUpdatingSketch(dblSk);
freeSpace = dblSk.levelsArr[0];
assert (freeSpace > 0);
}
//min and max change only after the compression, which may fail, has succeeded
dblSk.updateMinMax(item);
dblSk.incN(1);
dblSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
Expand Down
17 changes: 10 additions & 7 deletions src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger

// THE NEW ITEMS ARRAY
myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
? myCurFloatItemsArr
: new float[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];

//calculate the new levels array length
Expand All @@ -204,6 +197,16 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}

// THE NEW ITEMS ARRAY
//the levels array is allocated first, because this may be the current items array,
//and once it is overwritten nothing else may fail
myNewFloatItemsArr = (targetItemCount == myCurFloatItemsArr.length)
? myCurFloatItemsArr
: new float[targetItemCount];

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewFloatItemsArr, freeSpaceAtBottom, curItemCount);

//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,14 +334,15 @@ public void update(final float item) {

//Also Called from KllFloatsHelper::merge
static void updateFloat(final KllFloatsSketch fltSk, final float item) {
fltSk.updateMinMax(item);
int freeSpace = fltSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
KllFloatsHelper.compressWhileUpdatingSketch(fltSk);
freeSpace = fltSk.levelsArr[0];
assert (freeSpace > 0);
}
//min and max change only after the compression, which may fail, has succeeded
fltSk.updateMinMax(item);
fltSk.incN(1);
fltSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }

@Override
void incNumLevels() {
//the heap sketch computes num levels from the array itself, so this is not used on-heap
}

@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }

@Override
void incNumLevels() {
//the heap sketch computes num levels from the array itself, so this is not used on-heap
}

@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,10 +292,6 @@ MemorySegment getMemorySegment() {
@Override
void incN(final int increment) { n += increment; }

@Override
void incNumLevels() {
//the heap sketch computes num levels from the array itself, so this is not used on-heap
}

@Override
boolean isLevelZeroSorted() { return isLevelZeroSorted; }
Expand Down
1 change: 0 additions & 1 deletion src/main/java/org/apache/datasketches/kll/KllHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,6 @@ else if (sketchType == KLL_LONGS_SKETCH) {
myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2);
assert myNewLevelsArr.length == (myCurLevelsArr.length + 1);
myNewNumLevels = myCurNumLevels + 1;
sketch.incNumLevels(); //increment for off-heap
} else {
myNewLevelsArr = myCurLevelsArr;
myNewNumLevels = myCurNumLevels;
Expand Down
72 changes: 50 additions & 22 deletions src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,17 @@ static <T> T[] createItemsArray(final Class<T> clazz, final T item, final long w
private static <T> void compressWhileUpdatingSketch(final KllItemsSketch<T> itmSk) {
final int level =
findLevelToCompact(itmSk.getK(), itmSk.getM(), itmSk.getNumLevels(), itmSk.levelsArr);
//The comparator may throw while sorting or merging below. Nothing in the sketch changes before
//that work is done, except for a new top level, which is then removed again.
int[] savedLevelsArr = null;
Object[] savedItemsArr = null;
if (level == (itmSk.getNumLevels() - 1)) {
//The level to compact is the top level, thus we need to add a level.
//Be aware that this operation grows the items array,
//shifts the items data and the level boundaries of the data,
//and grows the levels array and increments numLevels_.
savedLevelsArr = itmSk.levelsArr;
savedItemsArr = itmSk.getTotalItemsArray(); //a copy; growth happens only O(log n) times
KllHelper.addEmptyTopLevelToCompletelyFullSketch(itmSk);
}
//after this point, the levelsArray will not be expanded, only modified.
Expand All @@ -83,18 +89,27 @@ private static <T> void compressWhileUpdatingSketch(final KllItemsSketch<T> itmS
final int halfAdjPop = adjPop / 2;

//the following is specific to generic Items
final Object[] myItemsArr = itmSk.getTotalItemsArray();
if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it
Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator);
}
if (popAbove == 0) {
KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
} else {
KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
KllItemsHelper.mergeSortedItemsArrays(
myItemsArr, adjBeg, halfAdjPop,
myItemsArr, rawEnd, popAbove,
myItemsArr, adjBeg + halfAdjPop, itmSk.comparator);
final Object[] myItemsArr;
try {
myItemsArr = itmSk.getTotalItemsArray(); //a copy, set into the sketch at the end
if (level == 0) { // level zero might not be sorted, so we must sort it if we wish to compact it
Arrays.sort((T[])myItemsArr, adjBeg, adjBeg + adjPop, itmSk.comparator);
}
if (popAbove == 0) {
KllItemsHelper.randomlyHalveUpItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
} else {
KllItemsHelper.randomlyHalveDownItems(myItemsArr, adjBeg, adjPop, KllSketch.random);
KllItemsHelper.mergeSortedItemsArrays(
myItemsArr, adjBeg, halfAdjPop,
myItemsArr, rawEnd, popAbove,
myItemsArr, adjBeg + halfAdjPop, itmSk.comparator);
}
} catch (final RuntimeException | Error e) {
if (savedLevelsArr != null) { //remove the new top level
itmSk.setLevelsArray(savedLevelsArr);
itmSk.setItemsArray(savedItemsArr);
}
throw e;
}

int newIndex = myLevelsArr[level + 1] - halfAdjPop; // adjust boundaries of the level above
Expand Down Expand Up @@ -189,14 +204,7 @@ static <T> void mergeItemImpl(final KllItemsSketch<T> mySketch,
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger

// THE NEW ITEMS ARRAY
myNewItemsArr = (targetItemCount == myCurItemsArr.length)
? myCurItemsArr
: new Object[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];

//calculate the new levels array length
Expand All @@ -210,6 +218,16 @@ static <T> void mergeItemImpl(final KllItemsSketch<T> mySketch,
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}

// THE NEW ITEMS ARRAY
//the levels array is allocated first, because this may be the current items array,
//and once it is overwritten nothing else may fail
myNewItemsArr = (targetItemCount == myCurItemsArr.length)
? myCurItemsArr
: new Object[targetItemCount];

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewItemsArr, freeSpaceAtBottom, curItemCount);

//MEMORY SPACE MANAGEMENT
//not used
//extra spaces to make comparison with other helpers easier
Expand Down Expand Up @@ -317,14 +335,25 @@ private static void randomlyHalveUpItems(final Object[] buf, final int start, fi

//Called from KllItemsSketch::update and this
static <T> void updateItem(final KllItemsSketch<T> itmSk, final Object item) {
itmSk.updateMinMax((T)item);
//compare first, but change min and max only after the compression, which may fail, has succeeded
final T newMin;
final T newMax;
if (itmSk.isEmpty()) {
newMin = (T) item;
newMax = (T) item;
} else {
newMin = (T) Util.minT(itmSk.getMinItem(), item, itmSk.comparator);
newMax = (T) Util.maxT(itmSk.getMaxItem(), item, itmSk.comparator);
}
int freeSpace = itmSk.levelsArr[0];
assert freeSpace >= 0;
if (freeSpace == 0) {
compressWhileUpdatingSketch(itmSk);
freeSpace = itmSk.levelsArr[0];
assert (freeSpace > 0);
}
itmSk.setMinItem(newMin);
itmSk.setMaxItem(newMax);
itmSk.incN(1);
itmSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
Expand All @@ -336,8 +365,7 @@ static <T> void updateItem(final KllItemsSketch<T> itmSk, final Object item) {
static <T> void updateItem(final KllItemsSketch<T> itmSk, final T item, final long weight) {
if (weight < itmSk.levelsArr[0]) {
for (int i = 0; i < (int)weight; i++) { updateItem(itmSk, item); }
} else {
itmSk.updateMinMax(item);
} else { //the merge updates min and max
final KllHeapItemsSketch<T> tmpSk =
new KllHeapItemsSketch<>(itmSk.getK(), DEFAULT_M, item, weight, itmSk.comparator, itmSk.serDe);
itmSk.merge(tmpSk);
Expand Down
14 changes: 0 additions & 14 deletions src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -378,10 +378,6 @@ int getTotalItemsNumBytes() {
throw new SketchesArgumentException(UNSUPPORTED_MSG);
}

@Override
void incNumLevels() {
//this is not used and must be a no-op.
}

abstract void setItemsArray(Object[] ItemsArr);

Expand All @@ -401,16 +397,6 @@ void setMemorySegment(final MemorySegment wseg) {
throw new SketchesArgumentException(UNSUPPORTED_MSG + "Sketch not writable.");
}

void updateMinMax(final T item) {
if (isEmpty()) {
setMinItem(item);
setMaxItem(item);
} else {
setMinItem(Util.minT(getMinItem(), item, comparator));
setMaxItem(Util.maxT(getMaxItem(), item, comparator));
}
}

private final ItemsSketchSortedView<T> refreshSortedView() {
if (itemsSV == null) {
final CreateSortedView csv = new CreateSortedView();
Expand Down
17 changes: 10 additions & 7 deletions src/main/java/org/apache/datasketches/kll/KllLongsHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,7 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o
myNewNumLevels = result[0];
assert myNewNumLevels <= ub; // ub may be much bigger

// THE NEW ITEMS ARRAY
myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length)
? myCurLongItemsArray
: new long[targetItemCount];
final int freeSpaceAtBottom = targetItemCount - curItemCount;

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount);
final int theShift = freeSpaceAtBottom - outlevels[0];

//calculate the new levels array length
Expand All @@ -204,6 +197,16 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o
myNewLevelsArr[lvl] = outlevels[lvl] + theShift;
}

// THE NEW ITEMS ARRAY
//the levels array is allocated first, because this may be the current items array,
//and once it is overwritten nothing else may fail
myNewLongItemsArray = (targetItemCount == myCurLongItemsArray.length)
? myCurLongItemsArray
: new long[targetItemCount];

//shift the new items array create space at bottom
System.arraycopy(workbuf, outlevels[0], myNewLongItemsArray, freeSpaceAtBottom, curItemCount);

//MemorySegment SPACE MANAGEMENT
if (mySketch.getMemorySegment() != null) {
final MemorySegment wseg = KllHelper.memorySegmentSpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,14 +334,15 @@ public void update(final long item) {

//Also Called from KllLongsHelper::merge
static void updateLong(final KllLongsSketch lngSk, final long item) {
lngSk.updateMinMax(item);
int freeSpace = lngSk.levelsArr[0];
assert (freeSpace >= 0);
if (freeSpace == 0) {
KllLongsHelper.compressWhileUpdatingSketch(lngSk);
freeSpace = lngSk.levelsArr[0];
assert (freeSpace > 0);
}
//min and max change only after the compression, which may fail, has succeeded
lngSk.updateMinMax(item);
lngSk.incN(1);
lngSk.setLevelZeroSorted(false);
final int nextPos = freeSpace - 1;
Expand Down
Loading
Loading