You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/03/26 02:26:05 UTC
[datasketches-java] 03/12: Interim 3
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch DirectKllDouble
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 032a9a6c53397c88ea0c6e3c2866df12f5f3f08b
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Mar 3 21:28:56 2022 -0800
Interim 3
---
.../org/apache/datasketches/kll/BaseKllSketch.java | 5 +
.../apache/datasketches/kll/KllDoublesSketch.java | 91 +++++++++++++++----
.../apache/datasketches/kll/KllFloatsSketch.java | 101 ++++++++++++++++-----
.../org/apache/datasketches/kll/KllHelper.java | 33 +++----
.../org/apache/datasketches/kll/PreambleUtil.java | 20 ++--
.../kll/KllDoublesSketchIteratorTest.java | 2 -
.../kll/KllFloatsSketchIteratorTest.java | 1 -
.../org/apache/datasketches/kll/KllHelperTest.java | 65 +++++++++++++
.../apache/datasketches/kll/MiscDoublesTest.java | 4 +-
.../apache/datasketches/kll/MiscFloatsTest.java | 4 +-
10 files changed, 244 insertions(+), 82 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java b/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
index 6dab928..e613982 100644
--- a/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
@@ -176,6 +176,11 @@ abstract class BaseKllSketch {
*/
public abstract byte[] toByteArray();
+ /**
+ * Returns serialized sketch in an updatable byte array form.
+ * @return serialized sketch in an updatable byte array form.
+ */
+ public abstract byte[] toUpdatableByteArray();
@Override
public String toString() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
index ec1e0a2..a95cc72 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
@@ -44,6 +44,7 @@ import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_EMPTY_FULL
import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_SINGLE;
import static org.apache.datasketches.kll.PreambleUtil.SER_VER_BYTE_ADR;
import static org.apache.datasketches.kll.PreambleUtil.SINGLE_ITEM_BIT_MASK;
+import static org.apache.datasketches.kll.PreambleUtil.UPDATABLE_BIT_MASK;
import java.util.Arrays;
@@ -65,6 +66,7 @@ public class KllDoublesSketch extends BaseKllSketch {
private double[] items_; // the continuous array of double items
private double minValue_;
private double maxValue_;
+ private static final boolean IS_DOUBLE = true;
/**
* Heap constructor with the default <em>k = 200</em>, which has a rank error of about 1.65%.
@@ -125,26 +127,31 @@ public class KllDoublesSketch extends BaseKllSketch {
n_ = 1;
minK_ = k_;
numLevels_ = 1;
- levels_ = new int[numLevels_ + 1]; //ALL
- final int offset = DATA_START_ADR_SINGLE_ITEM;
+ levels_ = new int[numLevels_ + 1];
final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
levels_[0] = itemCapacity - 1;
levels_[numLevels_] = itemCapacity; //load the last integer in levels_
items_ = new double[itemCapacity];
- mem.getDoubleArray(offset, items_, levels_[0], getNumRetained());
+ items_[levels_[0]] = mem.getDouble(DATA_START_ADR_SINGLE_ITEM);
minValue_ = items_[levels_[0]];
maxValue_ = items_[levels_[0]];
} else {
n_ = memChk.n;
minK_ = memChk.minK;
numLevels_ = memChk.numLevels;
- levels_ = new int[numLevels_ + 1]; //ALL
+ levels_ = new int[numLevels_ + 1];
int offset = DATA_START_ADR_DOUBLE;
final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
- // the last integer in levels_ is not serialized because it can be derived
- mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
- offset += numLevels_ * Integer.BYTES;
- levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+ if (memChk.updatable) {
+ // If updatable the last integer in levels_ IS serialized.
+ mem.getIntArray(offset, levels_, 0, numLevels_ + 1); //load levels_
+ offset += (numLevels_ + 1) * Integer.BYTES;
+ } else {
+ // If compact the last integer in levels_ is not serialized.
+ mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
+ offset += numLevels_ * Integer.BYTES;
+ levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+ }
minValue_ = mem.getDouble(offset);
offset += Double.BYTES;
maxValue_ = mem.getDouble(offset);
@@ -218,6 +225,8 @@ public class KllDoublesSketch extends BaseKllSketch {
return minValue_;
}
+ //Size related
+
/**
* Returns upper bound on the compact serialized size of a sketch given a parameter <em>k</em> and stream
* length. This method can be used if allocation of storage is necessary beforehand.
@@ -226,7 +235,7 @@ public class KllDoublesSketch extends BaseKllSketch {
* @return upper bound on the compact serialized size
*/
public static int getMaxSerializedSizeBytes(final int k, final long n) {
- final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, true);
+ final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, IS_DOUBLE);
return lvlStats.getCompactBytes();
}
@@ -236,7 +245,25 @@ public class KllDoublesSketch extends BaseKllSketch {
*/
public int getCurrentCompactSerializedSizeBytes() {
if (isEmpty()) { return N_LONG_ADR; }
- return KllHelper.getCompactSerializedSizeBytes(numLevels_, getNumRetained(), true);
+ return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, false);
+ }
+
+ /**
+ * Returns the current number of bytes this sketch would require to store in updatable form.
+ * @return the current number of bytes this sketch would require to store in updatable form.
+ */
+ public int getCurrentUpdatableSerializedSizeBytes() {
+ return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, true);
+ }
+
+ /**
+ * Returns the number of bytes this sketch would require to store.
+ * @return the number of bytes this sketch would require to store.
+ * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
+ */
+ @Deprecated
+ public int getSerializedSizeBytes() {
+ return getCurrentCompactSerializedSizeBytes();
}
/**
@@ -409,16 +436,6 @@ public class KllDoublesSketch extends BaseKllSketch {
}
/**
- * Returns the number of bytes this sketch would require to store.
- * @return the number of bytes this sketch would require to store.
- * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
- */
- @Deprecated
- public int getSerializedSizeBytes() {
- return getCurrentCompactSerializedSizeBytes();
- }
-
- /**
* @return the iterator for this class
*/
public KllDoublesSketchIterator iterator() {
@@ -494,6 +511,40 @@ public class KllDoublesSketch extends BaseKllSketch {
}
@Override
+ public byte[] toUpdatableByteArray() {
+ final int itemCap = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
+ final int numBytes = KllHelper.getSerializedSizeBytes(numLevels_, itemCap, IS_DOUBLE, true);
+ final byte[] bytes = new byte[numBytes];
+ final WritableMemory wmem = WritableMemory.writableWrap(bytes);
+ //load the preamble
+ wmem.putByte(PREAMBLE_INTS_BYTE_ADR, (byte) PREAMBLE_INTS_DOUBLE);
+ wmem.putByte(SER_VER_BYTE_ADR, SERIAL_VERSION_EMPTY_FULL);
+ wmem.putByte(FAMILY_BYTE_ADR, (byte) Family.KLL.getID());
+ final byte flags = (byte)
+ ((isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
+ | DOUBLES_SKETCH_BIT_MASK
+ | UPDATABLE_BIT_MASK);
+ wmem.putByte(FLAGS_BYTE_ADR, flags);
+ wmem.putShort(K_SHORT_ADR, (short) k_);
+ wmem.putByte(M_BYTE_ADR, (byte) m_);
+ //load data
+ wmem.putLong(N_LONG_ADR, n_);
+ wmem.putShort(MIN_K_SHORT_ADR, (short) minK_);
+ wmem.putByte(NUM_LEVELS_BYTE_ADR, (byte) numLevels_);
+ int offset = DATA_START_ADR_DOUBLE;
+ // the last integer in levels_ IS serialized
+ final int len = levels_.length;
+ wmem.putIntArray(offset, levels_, 0, len);
+ offset += len * Integer.BYTES;
+ wmem.putDouble(offset, minValue_);
+ offset += Double.BYTES;
+ wmem.putDouble(offset, maxValue_);
+ offset += Double.BYTES;
+ wmem.putDoubleArray(offset, items_, levels_[0], getNumRetained());
+ return bytes;
+ }
+
+ @Override
public String toString(final boolean withLevels, final boolean withData) {
final String epsPct = String.format("%.3f%%", getNormalizedRankError(false) * 100);
final String epsPMFPct = String.format("%.3f%%", getNormalizedRankError(true) * 100);
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
index f0aa48e..fc10591 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
@@ -43,6 +43,7 @@ import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_EMPTY_FULL
import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_SINGLE;
import static org.apache.datasketches.kll.PreambleUtil.SER_VER_BYTE_ADR;
import static org.apache.datasketches.kll.PreambleUtil.SINGLE_ITEM_BIT_MASK;
+import static org.apache.datasketches.kll.PreambleUtil.UPDATABLE_BIT_MASK;
import java.util.Arrays;
@@ -54,7 +55,7 @@ import org.apache.datasketches.kll.PreambleUtil.MemoryCheck;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;
-
+// (leave blank)
/**
* Please refer to the documentation in the package-info:<br>
* {@link org.apache.datasketches.kll}
@@ -65,6 +66,7 @@ public class KllFloatsSketch extends BaseKllSketch {
private float[] items_; // the continuous array of float items
private float minValue_;
private float maxValue_;
+ private static final boolean IS_DOUBLE = false;
/**
* Heap constructor with the default <em>k = 200</em>, which has a rank error of about 1.65%.
@@ -125,26 +127,31 @@ public class KllFloatsSketch extends BaseKllSketch {
n_ = 1;
minK_ = k_;
numLevels_ = 1;
- levels_ = new int[numLevels_ + 1]; //ALL
- final int offset = DATA_START_ADR_SINGLE_ITEM;
+ levels_ = new int[numLevels_ + 1];
final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
levels_[0] = itemCapacity - 1;
levels_[numLevels_] = itemCapacity; //load the last integer in levels_
items_ = new float[itemCapacity];
- mem.getFloatArray(offset, items_, levels_[0], getNumRetained());
+ items_[levels_[0]] = mem.getFloat(DATA_START_ADR_SINGLE_ITEM);
minValue_ = items_[levels_[0]];
maxValue_ = items_[levels_[0]];
} else {
n_ = memChk.n;
minK_ = memChk.minK;
numLevels_ = memChk.numLevels;
- levels_ = new int[numLevels_ + 1]; //ALL
+ levels_ = new int[numLevels_ + 1];
int offset = DATA_START_ADR_FLOAT;
final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
- // the last integer in levels_ is not serialized because it can be derived
- mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
- offset += numLevels_ * Integer.BYTES;
- levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+ if (memChk.updatable) {
+ // If updatable the last integer in levels_ IS serialized.
+ mem.getIntArray(offset, levels_, 0, numLevels_ + 1); //load levels_
+ offset += (numLevels_ + 1) * Integer.BYTES;
+ } else {
+ // If compact the last integer in levels_ is not serialized.
+ mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
+ offset += numLevels_ * Integer.BYTES;
+ levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+ }
minValue_ = mem.getFloat(offset);
offset += Float.BYTES;
maxValue_ = mem.getFloat(offset);
@@ -218,15 +225,17 @@ public class KllFloatsSketch extends BaseKllSketch {
return minValue_;
}
+ //Size related
+
/**
- * Returns upper bound on the serialized size of a sketch given a parameter <em>k</em> and stream
+ * Returns upper bound on the compact serialized size of a sketch given a parameter <em>k</em> and stream
* length. This method can be used if allocation of storage is necessary beforehand.
* @param k parameter that controls size of the sketch and accuracy of estimates
* @param n stream length
- * @return upper bound on the serialized size
+ * @return upper bound on the compact serialized size
*/
public static int getMaxSerializedSizeBytes(final int k, final long n) {
- final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, false);
+ final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, IS_DOUBLE);
return lvlStats.getCompactBytes();
}
@@ -236,7 +245,25 @@ public class KllFloatsSketch extends BaseKllSketch {
*/
public int getCurrentCompactSerializedSizeBytes() {
if (isEmpty()) { return N_LONG_ADR; }
- return KllHelper.getCompactSerializedSizeBytes(numLevels_, getNumRetained(), false);
+ return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, false);
+ }
+
+ /**
+ * Returns the current number of bytes this sketch would require to store in updatable form.
+ * @return the current number of bytes this sketch would require to store in updatable form.
+ */
+ public int getCurrentUpdatableSerializedSizeBytes() {
+ return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, true);
+ }
+
+ /**
+ * Returns the number of bytes this sketch would require to store.
+ * @return the number of bytes this sketch would require to store.
+ * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
+ */
+ @Deprecated
+ public int getSerializedSizeBytes() {
+ return getCurrentCompactSerializedSizeBytes();
}
/**
@@ -409,16 +436,6 @@ public class KllFloatsSketch extends BaseKllSketch {
}
/**
- * Returns the number of bytes this sketch would require to store.
- * @return the number of bytes this sketch would require to store.
- * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
- */
- @Deprecated
- public int getSerializedSizeBytes() {
- return getCurrentCompactSerializedSizeBytes();
- }
-
- /**
* @return the iterator for this class
*/
public KllFloatsSketchIterator iterator() {
@@ -455,7 +472,7 @@ public class KllFloatsSketch extends BaseKllSketch {
@Override
public byte[] toByteArray() {
- final byte[] bytes = new byte[getSerializedSizeBytes()];
+ final byte[] bytes = new byte[getCurrentCompactSerializedSizeBytes()];
final WritableMemory wmem = WritableMemory.writableWrap(bytes);
final boolean singleItem = n_ == 1;
final boolean empty = isEmpty();
@@ -468,7 +485,7 @@ public class KllFloatsSketch extends BaseKllSketch {
(empty ? EMPTY_BIT_MASK : 0)
| (isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
| (singleItem ? SINGLE_ITEM_BIT_MASK : 0));
-
+ // (leave blank)
wmem.putByte(FLAGS_BYTE_ADR, flags);
wmem.putShort(K_SHORT_ADR, (short) k_);
wmem.putByte(M_BYTE_ADR, (byte) m_);
@@ -494,6 +511,40 @@ public class KllFloatsSketch extends BaseKllSketch {
}
@Override
+ public byte[] toUpdatableByteArray() {
+ final int itemCap = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
+ final int numBytes = KllHelper.getSerializedSizeBytes(numLevels_, itemCap, IS_DOUBLE, true);
+ final byte[] bytes = new byte[numBytes];
+ final WritableMemory wmem = WritableMemory.writableWrap(bytes);
+ //load the preamble
+ wmem.putByte(PREAMBLE_INTS_BYTE_ADR, (byte) PREAMBLE_INTS_FLOAT);
+ wmem.putByte(SER_VER_BYTE_ADR, SERIAL_VERSION_EMPTY_FULL);
+ wmem.putByte(FAMILY_BYTE_ADR, (byte) Family.KLL.getID());
+ final byte flags = (byte)
+ ((isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
+ | UPDATABLE_BIT_MASK);
+ // (leave blank)
+ wmem.putByte(FLAGS_BYTE_ADR, flags);
+ wmem.putShort(K_SHORT_ADR, (short) k_);
+ wmem.putByte(M_BYTE_ADR, (byte) m_);
+ //load data
+ wmem.putLong(N_LONG_ADR, n_);
+ wmem.putShort(MIN_K_SHORT_ADR, (short) minK_);
+ wmem.putByte(NUM_LEVELS_BYTE_ADR, (byte) numLevels_);
+ int offset = DATA_START_ADR_FLOAT;
+ // the last integer in levels_ IS serialized
+ final int len = levels_.length;
+ wmem.putIntArray(offset, levels_, 0, len);
+ offset += len * Integer.BYTES;
+ wmem.putDouble(offset, minValue_);
+ offset += Float.BYTES;
+ wmem.putDouble(offset, maxValue_);
+ offset += Float.BYTES;
+ wmem.putFloatArray(offset, items_, levels_[0], getNumRetained());
+ return bytes;
+ }
+
+ @Override
public String toString(final boolean withLevels, final boolean withData) {
final String epsPct = String.format("%.3f%%", getNormalizedRankError(false) * 100);
final String epsPMFPct = String.format("%.3f%%", getNormalizedRankError(true) * 100);
diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java
index 891ed3c..2965d9f 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java
@@ -86,8 +86,8 @@ class KllHelper {
System.out.printf("%6d%,12d%8d%,16d\n", level, 1 << level, levelCap, maxNAtLevel);
}
}
- final int compactBytes = getCompactSerializedSizeBytes(numLevels, cumCap, isDouble);
- final int updatableBytes = getUpdatableSerializedSizeBytes(k, m, numLevels, isDouble);
+ final int compactBytes = getSerializedSizeBytes(numLevels, cumCap, isDouble, false);
+ final int updatableBytes = getSerializedSizeBytes(numLevels, cumCap, isDouble, true);
if (printDetail) {
System.out.printf(" TOTALS%10s %8d%,16d\n", "", cumCap, cumN);
System.out.println(" COMPACT BYTES: " + compactBytes);
@@ -135,28 +135,21 @@ class KllHelper {
public int getMaxCap() { return maxCap; }
}
- static int getCompactSerializedSizeBytes(final int numLevels, final int numRetained, final boolean isDouble) {
- if (numLevels == 1 && numRetained == 1) {
- return DATA_START_ADR_SINGLE_ITEM + (isDouble ? Double.BYTES : Float.BYTES);
- }
- // The last integer in levels_ is not serialized because it can be derived.
- // The + 2 is for min and max
- if (isDouble) {
- return DATA_START_ADR_DOUBLE + numLevels * Integer.BYTES + (numRetained + 2) * Double.BYTES;
+ static int getSerializedSizeBytes(final int numLevels, final int numRetained, final boolean isDouble,
+ final boolean updatable) {
+ int levelsBytes = 0;
+ if (!updatable) {
+ if (numLevels == 1 && numRetained == 1) {
+ return DATA_START_ADR_SINGLE_ITEM + (isDouble ? Double.BYTES : Float.BYTES);
+ }
+ levelsBytes = numLevels * Integer.BYTES;
} else {
- return DATA_START_ADR_FLOAT + numLevels * Integer.BYTES + (numRetained + 2) * Float.BYTES;
+ levelsBytes = (numLevels + 1) * Integer.BYTES;
}
- }
-
- static int getUpdatableSerializedSizeBytes(final int k, final int m, final int numLevels, final boolean isDouble) {
- //There are no special accommodations for empty or single item.
- //The last integer in levels IS serialized.
- // The + 2 is for min and max
- final int totCap = computeTotalItemCapacity(k, m, numLevels) + 2;
if (isDouble) {
- return DATA_START_ADR_DOUBLE + (numLevels + 1) * Integer.BYTES + totCap * Double.BYTES;
+ return DATA_START_ADR_DOUBLE + levelsBytes + (numRetained + 2) * Double.BYTES;
} else {
- return DATA_START_ADR_FLOAT + (numLevels + 1) * Integer.BYTES + totCap * Float.BYTES;
+ return DATA_START_ADR_FLOAT + levelsBytes + (numRetained + 2) * Float.BYTES;
}
}
diff --git a/src/main/java/org/apache/datasketches/kll/PreambleUtil.java b/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
index 3d9314b..1203d97 100644
--- a/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
@@ -133,13 +133,12 @@ final class PreambleUtil {
static final int LEVEL_ZERO_SORTED_BIT_MASK = 2;
static final int SINGLE_ITEM_BIT_MASK = 4;
static final int DOUBLES_SKETCH_BIT_MASK = 8;
- static final int UPDATABLE_BIT_MASK = 16;
+ static final int UPDATABLE_BIT_MASK = 16;
enum Layout {
FLOAT_FULL_COMPACT, FLOAT_EMPTY_COMPACT, FLOAT_SINGLE_COMPACT,
DOUBLE_FULL_COMPACT, DOUBLE_EMPTY_COMPACT, DOUBLE_SINGLE_COMPACT,
- FLOAT_FULL_NOT_COMPACT, FLOAT_EMPTY_NOT_COMPACT, FLOAT_SINGLE_NOT_COMPACT,
- DOUBLE_FULL_NOT_COMPACT, DOUBLE_EMPTY_NOT_COMPACT, DOUBLE_SINGLE_NOT_COMPACT }
+ FLOAT_UPDATABLE, DOUBLE_UPDATABLE }
/**
* Returns a human readable string summary of the internal state of the given byte array.
@@ -203,7 +202,7 @@ final class PreambleUtil {
m = extractM(srcMem);
KllHelper.checkK(k);
- if (m != 8) { System.err.println("WARNING: Minimum Level width set to non-default value: " + m); }
+ if (m != 8) { throwCustom(7, m); }
if (familyID != Family.KLL.getID()) { throwCustom(0, familyID); }
famName = idToFamily(familyID).toString();
if (famName != "KLL") { throwCustom(23, 0); }
@@ -215,7 +214,7 @@ final class PreambleUtil {
case 0: { //not empty, not single item, float full
if (preInts != PREAMBLE_INTS_FLOAT) { throwCustom(6, preInts); }
if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
- layout = updatable ? Layout.FLOAT_FULL_NOT_COMPACT : Layout.FLOAT_FULL_COMPACT;
+ layout = updatable ? Layout.FLOAT_UPDATABLE : Layout.FLOAT_FULL_COMPACT;
n = extractN(srcMem);
minK = extractMinK(srcMem);
numLevels = extractNumLevels(srcMem);
@@ -226,7 +225,7 @@ final class PreambleUtil {
if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
if (updatable) {
- layout = Layout.FLOAT_EMPTY_NOT_COMPACT;
+ layout = Layout.FLOAT_UPDATABLE;
n = extractN(srcMem);
if (n != 0) { throwCustom(21, (int) n); }
minK = extractMinK(srcMem);
@@ -245,7 +244,7 @@ final class PreambleUtil {
if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
if (serVer != SERIAL_VERSION_SINGLE) { throwCustom(4, serVer); }
if (updatable) {
- layout = Layout.FLOAT_SINGLE_NOT_COMPACT;
+ layout = Layout.FLOAT_UPDATABLE;
n = extractN(srcMem);
if (n != 1) { throwCustom(22, (int)n); }
minK = extractMinK(srcMem);
@@ -263,7 +262,7 @@ final class PreambleUtil {
case 8: { //not empty, not single item, double full
if (preInts != PREAMBLE_INTS_DOUBLE) { throwCustom(5, preInts); }
if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
- layout = updatable ? Layout.DOUBLE_FULL_NOT_COMPACT : Layout.DOUBLE_FULL_COMPACT;
+ layout = updatable ? Layout.DOUBLE_UPDATABLE : Layout.DOUBLE_FULL_COMPACT;
n = extractN(srcMem);
minK = extractMinK(srcMem);
numLevels = extractNumLevels(srcMem);
@@ -274,7 +273,7 @@ final class PreambleUtil {
if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
if (updatable) {
- layout = Layout.DOUBLE_EMPTY_NOT_COMPACT;
+ layout = Layout.DOUBLE_UPDATABLE;
n = extractN(srcMem);
if (n != 0) { throwCustom(21, (int) n); }
minK = extractMinK(srcMem);
@@ -293,7 +292,7 @@ final class PreambleUtil {
if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
if (serVer != SERIAL_VERSION_SINGLE) { throwCustom(4, serVer); }
if (updatable) {
- layout = Layout.DOUBLE_SINGLE_NOT_COMPACT;
+ layout = Layout.DOUBLE_UPDATABLE;
n = extractN(srcMem);
if (n != 1) { throwCustom(22, (int)n); }
minK = extractMinK(srcMem);
@@ -321,6 +320,7 @@ final class PreambleUtil {
case 4: msg = "Single Item Bit: 1 -> SerVer: " + SERIAL_VERSION_SINGLE + ", NOT: " + value; break;
case 5: msg = "Double Sketch Bit: 1 -> PreInts: " + PREAMBLE_INTS_DOUBLE + ", NOT: " + value; break;
case 6: msg = "Double Sketch Bit: 0 -> PreInts: " + PREAMBLE_INTS_FLOAT + ", NOT: " + value; break;
+ case 7: msg = "The M field must be set to " + DEFAULT_M + ", NOT: " + value; break;
case 20: msg = "Empty flag bit and SingleItem flag bit cannot both be set. Flags: " + value; break;
case 21: msg = "N != 0 and empty bit is set. N: " + value; break;
case 22: msg = "N != 1 and single item bit is set. N: " + value; break;
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
index 391052a..64a9950 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
@@ -61,6 +61,4 @@ public class KllDoublesSketchIteratorTest {
}
}
-
}
-
diff --git a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
index 7732efc..33d829f 100644
--- a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
@@ -22,7 +22,6 @@ package org.apache.datasketches.kll;
import org.testng.Assert;
import org.testng.annotations.Test;
-@SuppressWarnings("javadoc")
public class KllFloatsSketchIteratorTest {
@Test
diff --git a/src/test/java/org/apache/datasketches/kll/KllHelperTest.java b/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
index 4e56f86..b3e3969 100644
--- a/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
@@ -24,8 +24,10 @@ import static org.apache.datasketches.kll.KllHelper.getLevelStats;
import static org.testng.Assert.assertEquals;
import org.apache.datasketches.kll.KllHelper.LevelStats;
+import org.apache.datasketches.memory.Memory;
import org.testng.annotations.Test;
+@SuppressWarnings("unused")
public class KllHelperTest {
@Test //convert two false below to true for visual checking
@@ -45,4 +47,67 @@ public class KllHelperTest {
LevelStats lvlStats = getLevelStats(k, m, numLevels, false, false, true);
assertEquals(lvlStats.getCompactBytes(), 5708);
}
+
+ @Test
+ public void checkUpdatableSerDe() {
+ KllDoublesSketch sk = new KllDoublesSketch(200);
+ for (int i = 1; i <= 533; i++) { sk.update(i); }
+ int retained = sk.getNumRetained();
+ int numLevels = ((BaseKllSketch)sk).numLevels_;
+ println("NumLevels: " + numLevels);
+ println("NumRetained: " + retained);
+
+ byte[] compByteArr1 = sk.toByteArray();
+ int compBytes1 = compByteArr1.length;
+ println("compBytes1: " + compBytes1);
+
+ byte[] upByteArr1 = sk.toUpdatableByteArray();
+ int upBytes1 = upByteArr1.length;
+ println("upBytes1: " + upBytes1);
+
+ Memory mem;
+ KllDoublesSketch sk2;
+
+ mem = Memory.wrap(compByteArr1);
+ sk2 = KllDoublesSketch.heapify(mem);
+ byte[] compByteArr2 = sk2.toByteArray();
+ int compBytes2 = compByteArr2.length;
+ println("compBytes2: " + compBytes2);
+ assertEquals(compBytes1, compBytes2);
+ assertEquals(sk2.getNumRetained(), retained);
+
+ mem = Memory.wrap(compByteArr2);
+ sk2 = KllDoublesSketch.heapify(mem);
+ byte[] upByteArr2 = sk2.toUpdatableByteArray();
+ int upBytes2 = upByteArr2.length;
+ println("upBytes2: " + upBytes2);
+ assertEquals(upBytes1, upBytes2);
+ assertEquals(sk2.getNumRetained(), retained);
+ }
+
+ //Experimental
+
+ //@Test //convert two false below to true for visual checking
+ public void testGetAllLevelStats2() {
+ long n = 533;
+ int k = 200;
+ int m = 8;
+ LevelStats lvlStats = getAllLevelStatsGivenN(k, m, n, true, true, true);
+ }
+
+ //@Test
+ public void getStatsAtNumLevels2() {
+ int k = 20;
+ int m = 8;
+ int numLevels = 2;
+ LevelStats lvlStats = getLevelStats(k, m, numLevels, true, true, true);
+ }
+
+ /**
+ * Println Object o
+ * @param o object to print
+ */
+ static void println(Object o) {
+ //System.out.println(o.toString());
+ }
}
diff --git a/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java b/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
index 17e4e3d..1059489 100644
--- a/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
+++ b/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
@@ -64,11 +64,11 @@ public class MiscDoublesTest {
println("LB : " + lb);
}
- @Test
+ @Test(expectedExceptions = SketchesArgumentException.class)
public void checkHeapifyExceptions1() {
KllDoublesSketch sk = new KllDoublesSketch();
WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
- wmem.putByte(6, (byte)4); //use different M: produces a warning
+ wmem.putByte(6, (byte)4); //corrupt with different M
KllDoublesSketch.heapify(wmem);
}
diff --git a/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java b/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
index 77d5a5a..c684888 100644
--- a/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
+++ b/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
@@ -64,11 +64,11 @@ public class MiscFloatsTest {
println("LB : " + lb);
}
- @Test
+ @Test(expectedExceptions = SketchesArgumentException.class)
public void checkHeapifyExceptions1() {
KllFloatsSketch sk = new KllFloatsSketch();
WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
- wmem.putByte(6, (byte)4); //use different M: produces a warning
+ wmem.putByte(6, (byte)4); //corrupt with different M
KllFloatsSketch.heapify(wmem);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org