You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/03/26 02:26:05 UTC

[datasketches-java] 03/12: Interim 3

This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch DirectKllDouble
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git

commit 032a9a6c53397c88ea0c6e3c2866df12f5f3f08b
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Mar 3 21:28:56 2022 -0800

    Interim 3
---
 .../org/apache/datasketches/kll/BaseKllSketch.java |   5 +
 .../apache/datasketches/kll/KllDoublesSketch.java  |  91 +++++++++++++++----
 .../apache/datasketches/kll/KllFloatsSketch.java   | 101 ++++++++++++++++-----
 .../org/apache/datasketches/kll/KllHelper.java     |  33 +++----
 .../org/apache/datasketches/kll/PreambleUtil.java  |  20 ++--
 .../kll/KllDoublesSketchIteratorTest.java          |   2 -
 .../kll/KllFloatsSketchIteratorTest.java           |   1 -
 .../org/apache/datasketches/kll/KllHelperTest.java |  65 +++++++++++++
 .../apache/datasketches/kll/MiscDoublesTest.java   |   4 +-
 .../apache/datasketches/kll/MiscFloatsTest.java    |   4 +-
 10 files changed, 244 insertions(+), 82 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java b/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
index 6dab928..e613982 100644
--- a/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/BaseKllSketch.java
@@ -176,6 +176,11 @@ abstract class BaseKllSketch {
    */
   public abstract byte[] toByteArray();
 
+  /**
+   * Returns serialized sketch in an updatable byte array form.
+   * @return serialized sketch in an updatable byte array form.
+   */
+  public abstract byte[] toUpdatableByteArray();
 
   @Override
   public String toString() {
diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
index ec1e0a2..a95cc72 100644
--- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java
@@ -44,6 +44,7 @@ import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_EMPTY_FULL
 import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_SINGLE;
 import static org.apache.datasketches.kll.PreambleUtil.SER_VER_BYTE_ADR;
 import static org.apache.datasketches.kll.PreambleUtil.SINGLE_ITEM_BIT_MASK;
+import static org.apache.datasketches.kll.PreambleUtil.UPDATABLE_BIT_MASK;
 
 import java.util.Arrays;
 
@@ -65,6 +66,7 @@ public class KllDoublesSketch extends BaseKllSketch {
   private double[] items_; // the continuous array of double items
   private double minValue_;
   private double maxValue_;
+  private static final boolean IS_DOUBLE = true;
 
   /**
    * Heap constructor with the default <em>k = 200</em>, which has a rank error of about 1.65%.
@@ -125,26 +127,31 @@ public class KllDoublesSketch extends BaseKllSketch {
       n_ = 1;
       minK_ = k_;
       numLevels_ = 1;
-      levels_ = new int[numLevels_ + 1]; //ALL
-      final int offset = DATA_START_ADR_SINGLE_ITEM;
+      levels_ = new int[numLevels_ + 1];
       final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
       levels_[0] = itemCapacity - 1;
       levels_[numLevels_] = itemCapacity; //load the last integer in levels_
       items_ = new double[itemCapacity];
-      mem.getDoubleArray(offset, items_, levels_[0], getNumRetained());
+      items_[levels_[0]] = mem.getDouble(DATA_START_ADR_SINGLE_ITEM);
       minValue_ = items_[levels_[0]];
       maxValue_ = items_[levels_[0]];
     } else {
       n_ = memChk.n;
       minK_ = memChk.minK;
       numLevels_ = memChk.numLevels;
-      levels_ = new int[numLevels_ + 1]; //ALL
+      levels_ = new int[numLevels_ + 1];
       int offset = DATA_START_ADR_DOUBLE;
       final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
-      // the last integer in levels_ is not serialized because it can be derived
-      mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
-      offset += numLevels_ * Integer.BYTES;
-      levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+      if (memChk.updatable) {
+        // If updatable the last integer in levels_ IS serialized.
+        mem.getIntArray(offset, levels_, 0, numLevels_ + 1); //load levels_
+        offset += (numLevels_ + 1) * Integer.BYTES;
+      } else {
+        // If compact the last integer in levels_ is not serialized.
+        mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
+        offset += numLevels_ * Integer.BYTES;
+        levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+      }
       minValue_ = mem.getDouble(offset);
       offset += Double.BYTES;
       maxValue_ = mem.getDouble(offset);
@@ -218,6 +225,8 @@ public class KllDoublesSketch extends BaseKllSketch {
     return minValue_;
   }
 
+  //Size related
+
   /**
    * Returns upper bound on the compact serialized size of a sketch given a parameter <em>k</em> and stream
    * length. This method can be used if allocation of storage is necessary beforehand.
@@ -226,7 +235,7 @@ public class KllDoublesSketch extends BaseKllSketch {
    * @return upper bound on the compact serialized size
    */
   public static int getMaxSerializedSizeBytes(final int k, final long n) {
-    final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, true);
+    final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, IS_DOUBLE);
     return lvlStats.getCompactBytes();
   }
 
@@ -236,7 +245,25 @@ public class KllDoublesSketch extends BaseKllSketch {
    */
   public int getCurrentCompactSerializedSizeBytes() {
     if (isEmpty()) { return N_LONG_ADR; }
-    return KllHelper.getCompactSerializedSizeBytes(numLevels_, getNumRetained(), true);
+    return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, false);
+  }
+
+  /**
+   * Returns the current number of bytes this sketch would require to store in updatable form.
+   * @return the current number of bytes this sketch would require to store in updatable form.
+   */
+  public int getCurrentUpdatableSerializedSizeBytes() {
+    return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, true);
+  }
+
+  /**
+   * Returns the number of bytes this sketch would require to store.
+   * @return the number of bytes this sketch would require to store.
+   * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
+   */
+  @Deprecated
+  public int getSerializedSizeBytes() {
+    return getCurrentCompactSerializedSizeBytes();
   }
 
   /**
@@ -409,16 +436,6 @@ public class KllDoublesSketch extends BaseKllSketch {
   }
 
   /**
-   * Returns the number of bytes this sketch would require to store.
-   * @return the number of bytes this sketch would require to store.
-   * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
-   */
-  @Deprecated
-  public int getSerializedSizeBytes() {
-    return getCurrentCompactSerializedSizeBytes();
-  }
-
-  /**
    * @return the iterator for this class
    */
   public KllDoublesSketchIterator iterator() {
@@ -494,6 +511,40 @@ public class KllDoublesSketch extends BaseKllSketch {
   }
 
   @Override
+  public byte[] toUpdatableByteArray() {
+    final int itemCap = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
+    final int numBytes = KllHelper.getSerializedSizeBytes(numLevels_, itemCap, IS_DOUBLE, true);
+    final byte[] bytes = new byte[numBytes];
+    final WritableMemory wmem = WritableMemory.writableWrap(bytes);
+    //load the preamble
+    wmem.putByte(PREAMBLE_INTS_BYTE_ADR, (byte) PREAMBLE_INTS_DOUBLE);
+    wmem.putByte(SER_VER_BYTE_ADR, SERIAL_VERSION_EMPTY_FULL);
+    wmem.putByte(FAMILY_BYTE_ADR, (byte) Family.KLL.getID());
+    final byte flags = (byte)
+        ((isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
+        | DOUBLES_SKETCH_BIT_MASK
+        | UPDATABLE_BIT_MASK);
+    wmem.putByte(FLAGS_BYTE_ADR, flags);
+    wmem.putShort(K_SHORT_ADR, (short) k_);
+    wmem.putByte(M_BYTE_ADR, (byte) m_);
+    //load data
+    wmem.putLong(N_LONG_ADR, n_);
+    wmem.putShort(MIN_K_SHORT_ADR, (short) minK_);
+    wmem.putByte(NUM_LEVELS_BYTE_ADR, (byte) numLevels_);
+    int offset = DATA_START_ADR_DOUBLE;
+    // the last integer in levels_ IS serialized
+    final int len = levels_.length;
+    wmem.putIntArray(offset, levels_, 0, len);
+    offset += len * Integer.BYTES;
+    wmem.putDouble(offset, minValue_);
+    offset += Double.BYTES;
+    wmem.putDouble(offset, maxValue_);
+    offset += Double.BYTES;
+    wmem.putDoubleArray(offset, items_, levels_[0], getNumRetained());
+    return bytes;
+  }
+
+  @Override
   public String toString(final boolean withLevels, final boolean withData) {
     final String epsPct = String.format("%.3f%%", getNormalizedRankError(false) * 100);
     final String epsPMFPct = String.format("%.3f%%", getNormalizedRankError(true) * 100);
diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
index f0aa48e..fc10591 100644
--- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
+++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java
@@ -43,6 +43,7 @@ import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_EMPTY_FULL
 import static org.apache.datasketches.kll.PreambleUtil.SERIAL_VERSION_SINGLE;
 import static org.apache.datasketches.kll.PreambleUtil.SER_VER_BYTE_ADR;
 import static org.apache.datasketches.kll.PreambleUtil.SINGLE_ITEM_BIT_MASK;
+import static org.apache.datasketches.kll.PreambleUtil.UPDATABLE_BIT_MASK;
 
 import java.util.Arrays;
 
@@ -54,7 +55,7 @@ import org.apache.datasketches.kll.PreambleUtil.MemoryCheck;
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
 
-
+// (leave blank)
 /**
  * Please refer to the documentation in the package-info:<br>
  * {@link org.apache.datasketches.kll}
@@ -65,6 +66,7 @@ public class KllFloatsSketch extends BaseKllSketch {
   private float[] items_; // the continuous array of float items
   private float minValue_;
   private float maxValue_;
+  private static final boolean IS_DOUBLE = false;
 
   /**
    * Heap constructor with the default <em>k = 200</em>, which has a rank error of about 1.65%.
@@ -125,26 +127,31 @@ public class KllFloatsSketch extends BaseKllSketch {
       n_ = 1;
       minK_ = k_;
       numLevels_ = 1;
-      levels_ = new int[numLevels_ + 1]; //ALL
-      final int offset = DATA_START_ADR_SINGLE_ITEM;
+      levels_ = new int[numLevels_ + 1];
       final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
       levels_[0] = itemCapacity - 1;
       levels_[numLevels_] = itemCapacity; //load the last integer in levels_
       items_ = new float[itemCapacity];
-      mem.getFloatArray(offset, items_, levels_[0], getNumRetained());
+      items_[levels_[0]] = mem.getFloat(DATA_START_ADR_SINGLE_ITEM);
       minValue_ = items_[levels_[0]];
       maxValue_ = items_[levels_[0]];
     } else {
       n_ = memChk.n;
       minK_ = memChk.minK;
       numLevels_ = memChk.numLevels;
-      levels_ = new int[numLevels_ + 1]; //ALL
+      levels_ = new int[numLevels_ + 1];
       int offset = DATA_START_ADR_FLOAT;
       final int itemCapacity = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
-      // the last integer in levels_ is not serialized because it can be derived
-      mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
-      offset += numLevels_ * Integer.BYTES;
-      levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+      if (memChk.updatable) {
+        // If updatable the last integer in levels_ IS serialized.
+        mem.getIntArray(offset, levels_, 0, numLevels_ + 1); //load levels_
+        offset += (numLevels_ + 1) * Integer.BYTES;
+      } else {
+        // If compact the last integer in levels_ is not serialized.
+        mem.getIntArray(offset, levels_, 0, numLevels_); //load levels_
+        offset += numLevels_ * Integer.BYTES;
+        levels_[numLevels_] = itemCapacity; //load the last integer in levels_
+      }
       minValue_ = mem.getFloat(offset);
       offset += Float.BYTES;
       maxValue_ = mem.getFloat(offset);
@@ -218,15 +225,17 @@ public class KllFloatsSketch extends BaseKllSketch {
     return minValue_;
   }
 
+  //Size related
+
   /**
-   * Returns upper bound on the serialized size of a sketch given a parameter <em>k</em> and stream
+   * Returns upper bound on the compact serialized size of a sketch given a parameter <em>k</em> and stream
    * length. This method can be used if allocation of storage is necessary beforehand.
    * @param k parameter that controls size of the sketch and accuracy of estimates
    * @param n stream length
-   * @return upper bound on the serialized size
+   * @return upper bound on the compact serialized size
    */
   public static int getMaxSerializedSizeBytes(final int k, final long n) {
-    final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, false);
+    final LevelStats lvlStats = getAllLevelStatsGivenN(k, DEFAULT_M, n, false, false, IS_DOUBLE);
     return lvlStats.getCompactBytes();
   }
 
@@ -236,7 +245,25 @@ public class KllFloatsSketch extends BaseKllSketch {
    */
   public int getCurrentCompactSerializedSizeBytes() {
     if (isEmpty()) { return N_LONG_ADR; }
-    return KllHelper.getCompactSerializedSizeBytes(numLevels_, getNumRetained(), false);
+    return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, false);
+  }
+
+  /**
+   * Returns the current number of bytes this sketch would require to store in updatable form.
+   * @return the current number of bytes this sketch would require to store in updatable form.
+   */
+  public int getCurrentUpdatableSerializedSizeBytes() {
+    return KllHelper.getSerializedSizeBytes(numLevels_, getNumRetained(), IS_DOUBLE, true);
+  }
+
+  /**
+   * Returns the number of bytes this sketch would require to store.
+   * @return the number of bytes this sketch would require to store.
+   * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
+   */
+  @Deprecated
+  public int getSerializedSizeBytes() {
+    return getCurrentCompactSerializedSizeBytes();
   }
 
   /**
@@ -409,16 +436,6 @@ public class KllFloatsSketch extends BaseKllSketch {
   }
 
   /**
-   * Returns the number of bytes this sketch would require to store.
-   * @return the number of bytes this sketch would require to store.
-   * @deprecated use {@link #getCurrentCompactSerializedSizeBytes() }
-   */
-  @Deprecated
-  public int getSerializedSizeBytes() {
-    return getCurrentCompactSerializedSizeBytes();
-  }
-
-  /**
    * @return the iterator for this class
    */
   public KllFloatsSketchIterator iterator() {
@@ -455,7 +472,7 @@ public class KllFloatsSketch extends BaseKllSketch {
 
   @Override
   public byte[] toByteArray() {
-    final byte[] bytes = new byte[getSerializedSizeBytes()];
+    final byte[] bytes = new byte[getCurrentCompactSerializedSizeBytes()];
     final WritableMemory wmem = WritableMemory.writableWrap(bytes);
     final boolean singleItem = n_ == 1;
     final boolean empty = isEmpty();
@@ -468,7 +485,7 @@ public class KllFloatsSketch extends BaseKllSketch {
         (empty ? EMPTY_BIT_MASK : 0)
       | (isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
       | (singleItem ? SINGLE_ITEM_BIT_MASK : 0));
-
+    // (leave blank)
     wmem.putByte(FLAGS_BYTE_ADR, flags);
     wmem.putShort(K_SHORT_ADR, (short) k_);
     wmem.putByte(M_BYTE_ADR, (byte) m_);
@@ -494,6 +511,40 @@ public class KllFloatsSketch extends BaseKllSketch {
   }
 
   @Override
+  public byte[] toUpdatableByteArray() {
+    final int itemCap = KllHelper.computeTotalItemCapacity(k_, m_, numLevels_);
+    final int numBytes = KllHelper.getSerializedSizeBytes(numLevels_, itemCap, IS_DOUBLE, true);
+    final byte[] bytes = new byte[numBytes];
+    final WritableMemory wmem = WritableMemory.writableWrap(bytes);
+    //load the preamble
+    wmem.putByte(PREAMBLE_INTS_BYTE_ADR, (byte) PREAMBLE_INTS_FLOAT);
+    wmem.putByte(SER_VER_BYTE_ADR, SERIAL_VERSION_EMPTY_FULL);
+    wmem.putByte(FAMILY_BYTE_ADR, (byte) Family.KLL.getID());
+    final byte flags = (byte)
+        ((isLevelZeroSorted_ ? LEVEL_ZERO_SORTED_BIT_MASK : 0)
+        | UPDATABLE_BIT_MASK);
+    // (leave blank)
+    wmem.putByte(FLAGS_BYTE_ADR, flags);
+    wmem.putShort(K_SHORT_ADR, (short) k_);
+    wmem.putByte(M_BYTE_ADR, (byte) m_);
+    //load data
+    wmem.putLong(N_LONG_ADR, n_);
+    wmem.putShort(MIN_K_SHORT_ADR, (short) minK_);
+    wmem.putByte(NUM_LEVELS_BYTE_ADR, (byte) numLevels_);
+    int offset = DATA_START_ADR_FLOAT;
+    // the last integer in levels_ IS serialized
+    final int len = levels_.length;
+    wmem.putIntArray(offset, levels_, 0, len);
+    offset += len * Integer.BYTES;
+    wmem.putDouble(offset, minValue_);
+    offset += Float.BYTES;
+    wmem.putDouble(offset, maxValue_);
+    offset += Float.BYTES;
+    wmem.putFloatArray(offset, items_, levels_[0], getNumRetained());
+    return bytes;
+  }
+
+  @Override
   public String toString(final boolean withLevels, final boolean withData) {
     final String epsPct = String.format("%.3f%%", getNormalizedRankError(false) * 100);
     final String epsPMFPct = String.format("%.3f%%", getNormalizedRankError(true) * 100);
diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java
index 891ed3c..2965d9f 100644
--- a/src/main/java/org/apache/datasketches/kll/KllHelper.java
+++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java
@@ -86,8 +86,8 @@ class KllHelper {
         System.out.printf("%6d%,12d%8d%,16d\n", level, 1 << level, levelCap, maxNAtLevel);
       }
     }
-    final int compactBytes = getCompactSerializedSizeBytes(numLevels, cumCap, isDouble);
-    final int updatableBytes = getUpdatableSerializedSizeBytes(k, m, numLevels, isDouble);
+    final int compactBytes = getSerializedSizeBytes(numLevels, cumCap, isDouble, false);
+    final int updatableBytes = getSerializedSizeBytes(numLevels, cumCap, isDouble, true);
     if (printDetail) {
       System.out.printf(" TOTALS%10s %8d%,16d\n", "", cumCap, cumN);
       System.out.println(" COMPACT BYTES: " + compactBytes);
@@ -135,28 +135,21 @@ class KllHelper {
     public int getMaxCap() { return maxCap; }
   }
 
-  static int getCompactSerializedSizeBytes(final int numLevels, final int numRetained, final boolean isDouble) {
-    if (numLevels == 1 && numRetained == 1) {
-      return DATA_START_ADR_SINGLE_ITEM + (isDouble ? Double.BYTES : Float.BYTES);
-    }
-    // The last integer in levels_ is not serialized because it can be derived.
-    // The + 2 is for min and max
-    if (isDouble) {
-      return DATA_START_ADR_DOUBLE + numLevels * Integer.BYTES + (numRetained + 2) * Double.BYTES;
+  static int getSerializedSizeBytes(final int numLevels, final int numRetained, final boolean isDouble,
+      final boolean updatable) {
+    int levelsBytes = 0;
+    if (!updatable) {
+      if (numLevels == 1 && numRetained == 1) {
+        return DATA_START_ADR_SINGLE_ITEM + (isDouble ? Double.BYTES : Float.BYTES);
+      }
+      levelsBytes = numLevels * Integer.BYTES;
     } else {
-      return DATA_START_ADR_FLOAT + numLevels * Integer.BYTES + (numRetained + 2) * Float.BYTES;
+      levelsBytes = (numLevels + 1) * Integer.BYTES;
     }
-  }
-
-  static int getUpdatableSerializedSizeBytes(final int k, final int m, final int numLevels, final boolean isDouble) {
-    //There are no special accommodations for empty or single item.
-    //The last integer in levels IS serialized.
-    // The + 2 is for min and max
-    final int totCap = computeTotalItemCapacity(k, m, numLevels) + 2;
     if (isDouble) {
-      return DATA_START_ADR_DOUBLE + (numLevels + 1) * Integer.BYTES + totCap * Double.BYTES;
+      return DATA_START_ADR_DOUBLE + levelsBytes + (numRetained + 2) * Double.BYTES;
     } else {
-      return DATA_START_ADR_FLOAT + (numLevels + 1) * Integer.BYTES + totCap * Float.BYTES;
+      return DATA_START_ADR_FLOAT + levelsBytes + (numRetained + 2) * Float.BYTES;
     }
   }
 
diff --git a/src/main/java/org/apache/datasketches/kll/PreambleUtil.java b/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
index 3d9314b..1203d97 100644
--- a/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/kll/PreambleUtil.java
@@ -133,13 +133,12 @@ final class PreambleUtil {
   static final int LEVEL_ZERO_SORTED_BIT_MASK = 2;
   static final int SINGLE_ITEM_BIT_MASK       = 4;
   static final int DOUBLES_SKETCH_BIT_MASK    = 8;
-  static final int UPDATABLE_BIT_MASK       = 16;
+  static final int UPDATABLE_BIT_MASK         = 16;
 
   enum Layout {
     FLOAT_FULL_COMPACT,       FLOAT_EMPTY_COMPACT,      FLOAT_SINGLE_COMPACT,
     DOUBLE_FULL_COMPACT,      DOUBLE_EMPTY_COMPACT,     DOUBLE_SINGLE_COMPACT,
-    FLOAT_FULL_NOT_COMPACT,   FLOAT_EMPTY_NOT_COMPACT,  FLOAT_SINGLE_NOT_COMPACT,
-    DOUBLE_FULL_NOT_COMPACT,  DOUBLE_EMPTY_NOT_COMPACT, DOUBLE_SINGLE_NOT_COMPACT }
+    FLOAT_UPDATABLE,  DOUBLE_UPDATABLE }
 
   /**
    * Returns a human readable string summary of the internal state of the given byte array.
@@ -203,7 +202,7 @@ final class PreambleUtil {
       m = extractM(srcMem);
 
       KllHelper.checkK(k);
-      if (m != 8) { System.err.println("WARNING: Minimum Level width set to non-default value: " + m); }
+      if (m != 8) { throwCustom(7, m); }
       if (familyID != Family.KLL.getID()) { throwCustom(0, familyID); }
       famName = idToFamily(familyID).toString();
       if (famName != "KLL") { throwCustom(23, 0); }
@@ -215,7 +214,7 @@ final class PreambleUtil {
         case 0: { //not empty, not single item, float full
           if (preInts != PREAMBLE_INTS_FLOAT) { throwCustom(6, preInts); }
           if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
-          layout = updatable ? Layout.FLOAT_FULL_NOT_COMPACT : Layout.FLOAT_FULL_COMPACT;
+          layout = updatable ? Layout.FLOAT_UPDATABLE : Layout.FLOAT_FULL_COMPACT;
           n = extractN(srcMem);
           minK = extractMinK(srcMem);
           numLevels = extractNumLevels(srcMem);
@@ -226,7 +225,7 @@ final class PreambleUtil {
           if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
           if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
           if (updatable) {
-            layout = Layout.FLOAT_EMPTY_NOT_COMPACT;
+            layout = Layout.FLOAT_UPDATABLE;
             n = extractN(srcMem);
             if (n != 0) { throwCustom(21, (int) n); }
             minK = extractMinK(srcMem);
@@ -245,7 +244,7 @@ final class PreambleUtil {
           if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
           if (serVer != SERIAL_VERSION_SINGLE) { throwCustom(4, serVer); }
           if (updatable) {
-            layout = Layout.FLOAT_SINGLE_NOT_COMPACT;
+            layout = Layout.FLOAT_UPDATABLE;
             n = extractN(srcMem);
             if (n != 1) { throwCustom(22, (int)n); }
             minK = extractMinK(srcMem);
@@ -263,7 +262,7 @@ final class PreambleUtil {
         case 8: { //not empty, not single item, double full
           if (preInts != PREAMBLE_INTS_DOUBLE) { throwCustom(5, preInts); }
           if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
-          layout = updatable ? Layout.DOUBLE_FULL_NOT_COMPACT : Layout.DOUBLE_FULL_COMPACT;
+          layout = updatable ? Layout.DOUBLE_UPDATABLE : Layout.DOUBLE_FULL_COMPACT;
           n = extractN(srcMem);
           minK = extractMinK(srcMem);
           numLevels = extractNumLevels(srcMem);
@@ -274,7 +273,7 @@ final class PreambleUtil {
           if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
           if (serVer != SERIAL_VERSION_EMPTY_FULL) { throwCustom(2, serVer); }
           if (updatable) {
-            layout = Layout.DOUBLE_EMPTY_NOT_COMPACT;
+            layout = Layout.DOUBLE_UPDATABLE;
             n = extractN(srcMem);
             if (n != 0) { throwCustom(21, (int) n); }
             minK = extractMinK(srcMem);
@@ -293,7 +292,7 @@ final class PreambleUtil {
           if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { throwCustom(1, preInts); }
           if (serVer != SERIAL_VERSION_SINGLE) { throwCustom(4, serVer); }
           if (updatable) {
-            layout = Layout.DOUBLE_SINGLE_NOT_COMPACT;
+            layout = Layout.DOUBLE_UPDATABLE;
             n = extractN(srcMem);
             if (n != 1) { throwCustom(22, (int)n); }
             minK = extractMinK(srcMem);
@@ -321,6 +320,7 @@ final class PreambleUtil {
         case 4: msg = "Single Item Bit: 1 -> SerVer: " + SERIAL_VERSION_SINGLE + ", NOT: " + value; break;
         case 5: msg = "Double Sketch Bit: 1 -> PreInts: " + PREAMBLE_INTS_DOUBLE + ", NOT: " + value; break;
         case 6: msg = "Double Sketch Bit: 0 -> PreInts: " + PREAMBLE_INTS_FLOAT + ", NOT: " + value; break;
+        case 7: msg = "The M field must be set to " + DEFAULT_M + ", NOT: " + value; break;
         case 20: msg = "Empty flag bit and SingleItem flag bit cannot both be set. Flags: " + value; break;
         case 21: msg = "N != 0 and empty bit is set. N: " + value; break;
         case 22: msg = "N != 1 and single item bit is set. N: " + value; break;
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
index 391052a..64a9950 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchIteratorTest.java
@@ -61,6 +61,4 @@ public class KllDoublesSketchIteratorTest {
     }
   }
 
-
 }
-
diff --git a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
index 7732efc..33d829f 100644
--- a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchIteratorTest.java
@@ -22,7 +22,6 @@ package org.apache.datasketches.kll;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
-@SuppressWarnings("javadoc")
 public class KllFloatsSketchIteratorTest {
 
   @Test
diff --git a/src/test/java/org/apache/datasketches/kll/KllHelperTest.java b/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
index 4e56f86..b3e3969 100644
--- a/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllHelperTest.java
@@ -24,8 +24,10 @@ import static org.apache.datasketches.kll.KllHelper.getLevelStats;
 import static org.testng.Assert.assertEquals;
 
 import org.apache.datasketches.kll.KllHelper.LevelStats;
+import org.apache.datasketches.memory.Memory;
 import org.testng.annotations.Test;
 
+@SuppressWarnings("unused")
 public class KllHelperTest {
 
   @Test //convert two false below to true for visual checking
@@ -45,4 +47,67 @@ public class KllHelperTest {
     LevelStats lvlStats = getLevelStats(k, m, numLevels, false, false, true);
     assertEquals(lvlStats.getCompactBytes(), 5708);
   }
+
+  @Test
+  public void checkUpdatableSerDe() {
+    KllDoublesSketch sk = new KllDoublesSketch(200);
+    for (int i = 1; i <= 533; i++) { sk.update(i); }
+    int retained = sk.getNumRetained();
+    int numLevels = ((BaseKllSketch)sk).numLevels_;
+    println("NumLevels: " + numLevels);
+    println("NumRetained: " + retained);
+
+    byte[] compByteArr1 = sk.toByteArray();
+    int compBytes1 = compByteArr1.length;
+    println("compBytes1: " + compBytes1);
+
+    byte[] upByteArr1 = sk.toUpdatableByteArray();
+    int upBytes1 = upByteArr1.length;
+    println("upBytes1: " + upBytes1);
+
+    Memory mem;
+    KllDoublesSketch sk2;
+
+    mem = Memory.wrap(compByteArr1);
+    sk2 = KllDoublesSketch.heapify(mem);
+    byte[] compByteArr2 = sk2.toByteArray();
+    int compBytes2 = compByteArr2.length;
+    println("compBytes2: " + compBytes2);
+    assertEquals(compBytes1, compBytes2);
+    assertEquals(sk2.getNumRetained(), retained);
+
+    mem = Memory.wrap(compByteArr2);
+    sk2 = KllDoublesSketch.heapify(mem);
+    byte[] upByteArr2 = sk2.toUpdatableByteArray();
+    int upBytes2 = upByteArr2.length;
+    println("upBytes2: " + upBytes2);
+    assertEquals(upBytes1, upBytes2);
+    assertEquals(sk2.getNumRetained(), retained);
+  }
+
+  //Experimental
+
+  //@Test //convert two false below to true for visual checking
+  public void testGetAllLevelStats2() {
+    long n = 533;
+    int k = 200;
+    int m = 8;
+    LevelStats lvlStats = getAllLevelStatsGivenN(k, m, n, true, true, true);
+  }
+
+  //@Test
+  public void getStatsAtNumLevels2() {
+    int k = 20;
+    int m = 8;
+    int numLevels = 2;
+    LevelStats lvlStats = getLevelStats(k, m, numLevels, true, true, true);
+  }
+
+  /**
+   * Println Object o
+   * @param o object to print
+   */
+  static void println(Object o) {
+    //System.out.println(o.toString());
+  }
 }
diff --git a/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java b/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
index 17e4e3d..1059489 100644
--- a/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
+++ b/src/test/java/org/apache/datasketches/kll/MiscDoublesTest.java
@@ -64,11 +64,11 @@ public class MiscDoublesTest {
     println("LB      : " + lb);
   }
 
-  @Test
+  @Test(expectedExceptions = SketchesArgumentException.class)
   public void checkHeapifyExceptions1() {
     KllDoublesSketch sk = new KllDoublesSketch();
     WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
-    wmem.putByte(6, (byte)4); //use different M: produces a warning
+    wmem.putByte(6, (byte)4); //corrupt with different M
     KllDoublesSketch.heapify(wmem);
   }
 
diff --git a/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java b/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
index 77d5a5a..c684888 100644
--- a/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
+++ b/src/test/java/org/apache/datasketches/kll/MiscFloatsTest.java
@@ -64,11 +64,11 @@ public class MiscFloatsTest {
     println("LB      : " + lb);
   }
 
-  @Test
+  @Test(expectedExceptions = SketchesArgumentException.class)
   public void checkHeapifyExceptions1() {
     KllFloatsSketch sk = new KllFloatsSketch();
     WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
-    wmem.putByte(6, (byte)4); //use different M: produces a warning
+    wmem.putByte(6, (byte)4); //corrupt with different M
     KllFloatsSketch.heapify(wmem);
   }
 

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org