You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2019/12/24 07:11:35 UTC

[incubator-datasketches-java] branch NewHllIterator updated (94afdb8 -> 838976b)

This is an automated email from the ASF dual-hosted git repository.

leerho pushed a change to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git.


    from 94afdb8  Interim update
     new f2f7ec5  Interim update
     new be8dbfe  Interim
     new b042a2e  restore to no checkReadOnly on DirectHllArray / putAuxHashMap().
     new a5bf06a  Remove troubleshooting tests
     new e5b8a42  Interim updates
     new 93e4ca2  Added Isomorphism tests, and a few others.
     new d450b47  Added isomorphic tests.
     new 72e91b4  improve performance of integer and long byte conversions.
     new 1238051  Minor updates to LICENSE
     add 01d3841  update hll documentation
     add 34c0d93  Merge pull request #286 from apache/hll_doc_update
     new 838976b  Merge branch 'master' into NewHllIterator

The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 LICENSE                                            |  18 +-
 src/main/java/org/apache/datasketches/Util.java    |  46 +--
 .../apache/datasketches/hll/AbstractCoupons.java   |   8 +
 .../apache/datasketches/hll/AbstractHllArray.java  |   3 +-
 .../org/apache/datasketches/hll/BaseHllSketch.java |   4 +-
 .../org/apache/datasketches/hll/Conversions.java   |  16 +-
 .../apache/datasketches/hll/DirectAuxHashMap.java  |   9 +-
 .../apache/datasketches/hll/DirectCouponList.java  |   2 -
 .../apache/datasketches/hll/DirectHllArray.java    |  34 ++-
 .../java/org/apache/datasketches/hll/HllArray.java |  13 +
 .../org/apache/datasketches/hll/HllEstimators.java |  25 +-
 .../apache/datasketches/hll/HllPairIterator.java   |   2 -
 .../org/apache/datasketches/hll/HllSketch.java     |   8 +-
 .../org/apache/datasketches/hll/HllSketchImpl.java |   4 +
 .../org/apache/datasketches/hll/PreambleUtil.java  |  13 +
 .../apache/datasketches/hll/ToByteArrayImpl.java   |   2 +
 .../java/org/apache/datasketches/hll/Union.java    | 125 ++++++--
 .../apache/datasketches/hll/CouponListTest.java    |   9 +-
 .../org/apache/datasketches/hll/HllArrayTest.java  |  10 +-
 .../apache/datasketches/hll/IsomorphicTest.java    | 317 +++++++++++++++++++++
 .../org/apache/datasketches/hll/MiscTests.java     |  72 -----
 .../datasketches/hll/ToFromByteArrayTest.java      |  10 +-
 .../org/apache/datasketches/hll/UnionCaseTest.java |  51 +++-
 23 files changed, 616 insertions(+), 185 deletions(-)
 create mode 100644 src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
 delete mode 100644 src/test/java/org/apache/datasketches/hll/MiscTests.java


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 09/10: Minor updates to LICENSE

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit 12380510a3f975f7ebaeca79f737910ad610c1be
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Dec 23 16:59:12 2019 -0800

    Minor updates to LICENSE
---
 LICENSE | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/LICENSE b/LICENSE
index 720d42f..6c89e81 100644
--- a/LICENSE
+++ b/LICENSE
@@ -225,7 +225,7 @@ APPENDIX B: Additional licenses relevant to this product:
     Code locations:
     -------------------------------------------------------------
     This product contains code to implement the xxHash function:
-      * https://github.com/apache/incubator-datasketches-java/blob/master/src/main/java/org/apache/datasketches/hash/XxHash.java
+      * src/main/java/org/apache/datasketches/hash/XxHash.java
     and adapted from Java source code located at:
       * https://github.com/OpenHFT/Zero-Allocation-Hashing/blob/master/src/main/java/net/openhft/hashing/XxHash.java
 
@@ -264,8 +264,8 @@ APPENDIX B: Additional licenses relevant to this product:
     Code locations:
     -------------------------------------------------------------
     This product contains code to implement the xxHash function:
-      * https://github.com/apache/incubator-datasketches-java/blob/master/src/main/java/org/apache/datasketches/hash/XxHash.java
-      * https://github.com/apache/incubator-datasketches-java/blob/master/src/test/java/org/apache/datasketches/hash/XxHashTest.java
+      * src/main/java/org/apache/datasketches/hash/XxHash.java
+      * src/test/java/org/apache/datasketches/hash/XxHashTest.java
     and adapted from C++ source code located at:
       * https://github.com/Cyan4973/xxHash/blob/dev/xxhash.c
       * https://github.com/Cyan4973/xxHash/blob/dev/xxhash.h
@@ -277,12 +277,12 @@ APPENDIX B: Additional licenses relevant to this product:
     =============================================================
     This product contains code to implement and test Austin Appleby's C++ public domain implementation of the 
     MurmurHash3 algorithm:
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/main/java/org/apache/datasketches/hash/MurmurHash3.java
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/main/java/org/apache/datasketches/hash/MurmurHash3v2.java
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java
-    * https://github.com/apache/incubator-datasketches-java/blob/master/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java
+    * src/main/java/org/apache/datasketches/hash/MurmurHash3.java
+    * src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java
+    * src/main/java/org/apache/datasketches/hash/MurmurHash3v2.java
+    * src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
+    * src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java
+    * src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java
     and adapted from C++ source code located at:
     * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 01/10: Interim update

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit f2f7ec598677cc1d8d708325bb4b048f10057c97
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Wed Dec 18 18:04:59 2019 -0800

    Interim update
---
 .../apache/datasketches/hll/AbstractCoupons.java   |  8 +++
 .../apache/datasketches/hll/AbstractHllArray.java  |  3 +-
 .../org/apache/datasketches/hll/BaseHllSketch.java |  4 +-
 .../org/apache/datasketches/hll/Conversions.java   | 16 +++--
 .../apache/datasketches/hll/DirectHllArray.java    | 33 +++++++++-
 .../java/org/apache/datasketches/hll/HllArray.java | 11 ++++
 .../org/apache/datasketches/hll/HllEstimators.java | 25 +-------
 .../org/apache/datasketches/hll/HllSketchImpl.java |  4 ++
 .../org/apache/datasketches/hll/PreambleUtil.java  | 13 ++++
 .../java/org/apache/datasketches/hll/Union.java    | 75 +++++++++++++++++++---
 .../datasketches/hll/ToFromByteArrayTest.java      | 12 ++--
 11 files changed, 160 insertions(+), 44 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java b/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java
index bb01e13..93a2e32 100644
--- a/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java
+++ b/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java
@@ -105,9 +105,17 @@ abstract class AbstractCoupons extends HllSketchImpl {
   }
 
   @Override
+  boolean isRebuildCurMinNumKxQFlag() {
+    return false;
+  }
+
+  @Override
   void putEmptyFlag(final boolean empty) {} //no-op for coupons
 
   @Override
+  void putRebuildCurMinNumKxQFlag(final boolean rebuild) {} //no-op for coupons
+
+  @Override
   byte[] toCompactByteArray() {
     return toCouponByteArray(this, true);
   }
diff --git a/src/main/java/org/apache/datasketches/hll/AbstractHllArray.java b/src/main/java/org/apache/datasketches/hll/AbstractHllArray.java
index 575fe7f..9f2c630 100644
--- a/src/main/java/org/apache/datasketches/hll/AbstractHllArray.java
+++ b/src/main/java/org/apache/datasketches/hll/AbstractHllArray.java
@@ -78,7 +78,7 @@ abstract class AbstractHllArray extends HllSketchImpl {
   //In C: again-two-registers.c hhb_get_composite_estimate L1489
   @Override
   double getCompositeEstimate() {
-    return HllEstimators.hllCompositeEstimate(this, isOutOfOrderFlag());
+    return HllEstimators.hllCompositeEstimate(this);
   }
 
   abstract int getCurMin();
@@ -208,6 +208,7 @@ abstract class AbstractHllArray extends HllSketchImpl {
 
   /**
    * Common HIP and KxQ incremental update for all heap and direct Hll.
+   * This is used when incrementally updating an existing array with non-zero values.
    * @param host the origin implementation
    * @param oldValue old value
    * @param newValue new value
diff --git a/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java b/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java
index 844f328..3a1ec80 100644
--- a/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java
+++ b/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java
@@ -35,6 +35,8 @@ import org.apache.datasketches.memory.Memory;
  */
 abstract class BaseHllSketch {
 
+  abstract void couponUpdate(int coupon);
+
   /**
    * Gets the size in bytes of the current sketch when serialized using
    * <i>toCompactByteArray()</i>.
@@ -375,6 +377,6 @@ abstract class BaseHllSketch {
     return (value << KEY_BITS_26) | addr26;
   }
 
-  abstract void couponUpdate(int coupon);
+
 
 }
diff --git a/src/main/java/org/apache/datasketches/hll/Conversions.java b/src/main/java/org/apache/datasketches/hll/Conversions.java
index 9983e6a..35eaa0a 100644
--- a/src/main/java/org/apache/datasketches/hll/Conversions.java
+++ b/src/main/java/org/apache/datasketches/hll/Conversions.java
@@ -41,7 +41,8 @@ class Conversions {
     final int curMin = HllUtil.getValue(pair);
     final int numAtCurMin = HllUtil.getLow26(pair);
 
-    //2nd pass: Must know curMin. Populate KxQ registers, build AuxHashMap if needed
+    //2nd pass: Must know curMin to create AuxHashMap.
+    //Populate KxQ registers, build AuxHashMap if needed
     final PairIterator itr = srcHllArr.iterator();
     AuxHashMap auxHashMap = hll4Array.getAuxHashMap(); //may be null
     while (itr.nextValid()) {
@@ -66,16 +67,23 @@ class Conversions {
     return hll4Array;
   }
 
-  static final int curMinAndNum(final AbstractHllArray hllArr) {
+  /**
+   * If the given absHllArr is type HLL_4, this returns the correct curMin and numAtCurMin.
+   * For HLL_6 and HLL_8, curMin is always 0, and numAtCurMin is the number of zero slots.
+   * @param absHllArr an instance of AbstractHllArray
+   * @return pair values representing numAtCurMin and curMin
+   */
+  private static final int curMinAndNum(final AbstractHllArray absHllArr) {
     int curMin = 64;
     int numAtCurMin = 0;
-    final PairIterator itr = hllArr.iterator();
+    final PairIterator itr = absHllArr.iterator();
     while (itr.nextAll()) {
       final int v = itr.getValue();
+      if (v > curMin) { continue; }
       if (v < curMin) {
         curMin = v;
         numAtCurMin = 1;
-      } else if (v == curMin) { //missing else
+      } else {
         numAtCurMin++;
       }
     }
diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
index 48f798f..60e7566 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
@@ -31,6 +31,7 @@ import static org.apache.datasketches.hll.PreambleUtil.extractKxQ1;
 import static org.apache.datasketches.hll.PreambleUtil.extractLgK;
 import static org.apache.datasketches.hll.PreambleUtil.extractNumAtCurMin;
 import static org.apache.datasketches.hll.PreambleUtil.extractOooFlag;
+import static org.apache.datasketches.hll.PreambleUtil.extractRebuildCurMinNumKxQFlag;
 import static org.apache.datasketches.hll.PreambleUtil.extractTgtHllType;
 import static org.apache.datasketches.hll.PreambleUtil.insertAuxCount;
 import static org.apache.datasketches.hll.PreambleUtil.insertCompactFlag;
@@ -42,6 +43,7 @@ import static org.apache.datasketches.hll.PreambleUtil.insertKxQ1;
 import static org.apache.datasketches.hll.PreambleUtil.insertLgArr;
 import static org.apache.datasketches.hll.PreambleUtil.insertNumAtCurMin;
 import static org.apache.datasketches.hll.PreambleUtil.insertOooFlag;
+import static org.apache.datasketches.hll.PreambleUtil.insertRebuildCurMinNumKxQFlag;
 
 import org.apache.datasketches.SketchesArgumentException;
 import org.apache.datasketches.memory.Memory;
@@ -88,12 +90,14 @@ abstract class DirectHllArray extends AbstractHllArray {
 
   @Override
   void addToHipAccum(final double delta) {
+    checkReadOnly(wmem);
     final double hipAccum = mem.getDouble(HIP_ACCUM_DOUBLE);
     wmem.putDouble(HIP_ACCUM_DOUBLE, hipAccum + delta);
   }
 
   @Override
   void decNumAtCurMin() {
+    checkReadOnly(wmem);
     int numAtCurMin = mem.getInt(CUR_MIN_COUNT_INT);
     wmem.putInt(CUR_MIN_COUNT_INT, --numAtCurMin);
   }
@@ -179,7 +183,13 @@ abstract class DirectHllArray extends AbstractHllArray {
   }
 
   @Override
+  boolean isRebuildCurMinNumKxQFlag() {
+    return extractRebuildCurMinNumKxQFlag(mem);
+  }
+
+  @Override
   void putAuxHashMap(final AuxHashMap auxHashMap, final boolean compact) {
+    checkReadOnly(wmem);
     if (auxHashMap instanceof HeapAuxHashMap) {
       if (compact) {
         this.auxHashMap = auxHashMap; //heap and compact
@@ -198,39 +208,52 @@ abstract class DirectHllArray extends AbstractHllArray {
 
   @Override
   void putCurMin(final int curMin) {
+    checkReadOnly(wmem);
     insertCurMin(wmem, curMin);
   }
 
   @Override
   void putEmptyFlag(final boolean empty) {
+    checkReadOnly(wmem);
     insertEmptyFlag(wmem, empty);
   }
 
   @Override
   void putHipAccum(final double hipAccum) {
+    checkReadOnly(wmem);
     insertHipAccum(wmem, hipAccum);
   }
 
   @Override
   void putKxQ0(final double kxq0) {
+    checkReadOnly(wmem);
     insertKxQ0(wmem, kxq0);
   }
 
   @Override //called very very very rarely
   void putKxQ1(final double kxq1) {
+    checkReadOnly(wmem);
     insertKxQ1(wmem, kxq1);
   }
 
   @Override
   void putNumAtCurMin(final int numAtCurMin) {
+    checkReadOnly(wmem);
     insertNumAtCurMin(wmem, numAtCurMin);
   }
 
   @Override //not used on the direct side
   void putOutOfOrderFlag(final boolean oooFlag) {
+    checkReadOnly(wmem);
     insertOooFlag(wmem, oooFlag);
   }
 
+  @Override
+  void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+    checkReadOnly(wmem);
+    insertRebuildCurMinNumKxQFlag(wmem, rebuild);
+  }
+
   @Override //used by HLL6 and HLL8, overridden by HLL4
   byte[] toCompactByteArray() {
     return toUpdatableByteArray(); //indistinguishable for HLL6 and HLL8
@@ -248,12 +271,16 @@ abstract class DirectHllArray extends AbstractHllArray {
 
   @Override
   HllSketchImpl reset() {
-    if (wmem == null) {
-      throw new SketchesArgumentException("Cannot reset a read-only sketch");
-    }
+    checkReadOnly(wmem);
     insertEmptyFlag(wmem, true);
     final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
     wmem.clear(0, bytes);
     return DirectCouponList.newInstance(lgConfigK, tgtHllType, wmem);
   }
+
+  private static final void checkReadOnly(final WritableMemory wmem) {
+    if (wmem == null) {
+      throw new SketchesArgumentException("Cannot modify a read-only sketch");
+    }
+  }
 }
diff --git a/src/main/java/org/apache/datasketches/hll/HllArray.java b/src/main/java/org/apache/datasketches/hll/HllArray.java
index 7772070..37cbbe7 100644
--- a/src/main/java/org/apache/datasketches/hll/HllArray.java
+++ b/src/main/java/org/apache/datasketches/hll/HllArray.java
@@ -40,6 +40,7 @@ import org.apache.datasketches.memory.WritableMemory;
  */
 abstract class HllArray extends AbstractHllArray {
   boolean oooFlag = false; //Out-Of-Order Flag
+  boolean rebuildCurMinNumKxQ = false;
   int curMin; //always zero for Hll6 and Hll8, only used by Hll4Array
   int numAtCurMin; //# of values at curMin. If curMin = 0, it is # of zeros
   double hipAccum;
@@ -182,6 +183,11 @@ abstract class HllArray extends AbstractHllArray {
   }
 
   @Override
+  boolean isRebuildCurMinNumKxQFlag() {
+    return rebuildCurMinNumKxQ;
+  }
+
+  @Override
   void putAuxHashMap(final AuxHashMap auxHashMap, final boolean compact) {
     this.auxHashMap = auxHashMap;
   }
@@ -220,6 +226,11 @@ abstract class HllArray extends AbstractHllArray {
   }
 
   @Override
+  void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+    rebuildCurMinNumKxQ = rebuild;
+  }
+
+  @Override
   HllSketchImpl reset() {
     return new CouponList(lgConfigK, tgtHllType, CurMode.LIST);
   }
diff --git a/src/main/java/org/apache/datasketches/hll/HllEstimators.java b/src/main/java/org/apache/datasketches/hll/HllEstimators.java
index 46c4cee..78df59b 100644
--- a/src/main/java/org/apache/datasketches/hll/HllEstimators.java
+++ b/src/main/java/org/apache/datasketches/hll/HllEstimators.java
@@ -19,7 +19,6 @@
 
 package org.apache.datasketches.hll;
 
-import static org.apache.datasketches.Util.invPow2;
 import static org.apache.datasketches.hll.HllUtil.HLL_HIP_RSE_FACTOR;
 import static org.apache.datasketches.hll.HllUtil.HLL_NON_HIP_RSE_FACTOR;
 import static org.apache.datasketches.hll.HllUtil.MIN_LOG_K;
@@ -96,14 +95,9 @@ class HllEstimators {
    * @return the composite estimate
    */
   //In C: again-two-registers.c hhb_get_composite_estimate L1489
-  static final double hllCompositeEstimate(final AbstractHllArray absHllArr,
-      final boolean rebuildKxQ) {
+  static final double hllCompositeEstimate(final AbstractHllArray absHllArr) {
     final int lgConfigK = absHllArr.getLgConfigK();
-    final double kxqSum;
-    if (rebuildKxQ) { kxqSum = computeKxQSum(absHllArr); }
-    else { kxqSum = absHllArr.getKxQ0() + absHllArr.getKxQ1(); }
-
-    final double rawEst = getHllRawEstimate(lgConfigK, kxqSum);
+    final double rawEst = getHllRawEstimate(lgConfigK, absHllArr.getKxQ0() + absHllArr.getKxQ1());
 
     final double[] xArr = CompositeInterpolationXTable.xArrs[lgConfigK - MIN_LOG_K];
     final double yStride = CompositeInterpolationXTable.yStrides[lgConfigK - MIN_LOG_K];
@@ -153,7 +147,7 @@ class HllEstimators {
    * Refer to Wikipedia: Coupon Collector Problem
    * @param lgConfigK the current configured lgK of the sketch
    * @param curMin the current minimum value of the HLL window
-   * @param numAtCurMin the current number of rows with the value curMin
+   * @param numAtCurMin the current number of slots with the value curMin
    * @return the very low range estimate
    */
   //In C: again-two-registers.c hhb_get_improved_linear_counting_estimate L1274
@@ -184,17 +178,4 @@ class HllEstimators {
     return hyperEst;
   }
 
-  static final double computeKxQSum(final AbstractHllArray host) {
-    final int configK = 1 << host.getLgConfigK();
-    final PairIterator itr = host.iterator();
-    double kxq0 = configK;
-    double kxq1 = 0;
-    while (itr.nextValid()) {
-      final int actualValue = itr.getValue();
-      if (actualValue < 32) { kxq0 += invPow2(actualValue) - 1.0; }
-      else                  { kxq1 += invPow2(actualValue) - 1.0; }
-    }
-    return kxq0 + kxq1;
-  }
-
 }
diff --git a/src/main/java/org/apache/datasketches/hll/HllSketchImpl.java b/src/main/java/org/apache/datasketches/hll/HllSketchImpl.java
index 6465403..dde936d 100644
--- a/src/main/java/org/apache/datasketches/hll/HllSketchImpl.java
+++ b/src/main/java/org/apache/datasketches/hll/HllSketchImpl.java
@@ -96,6 +96,8 @@ abstract class HllSketchImpl {
 
   abstract boolean isOutOfOrderFlag();
 
+  abstract boolean isRebuildCurMinNumKxQFlag();
+
   abstract boolean isSameResource(Memory mem);
 
   abstract PairIterator iterator();
@@ -106,6 +108,8 @@ abstract class HllSketchImpl {
 
   abstract void putOutOfOrderFlag(boolean oooFlag);
 
+  abstract void putRebuildCurMinNumKxQFlag(boolean rebuild);
+
   abstract HllSketchImpl reset();
 
   abstract byte[] toCompactByteArray();
diff --git a/src/main/java/org/apache/datasketches/hll/PreambleUtil.java b/src/main/java/org/apache/datasketches/hll/PreambleUtil.java
index 070f27b..93e3d7b 100644
--- a/src/main/java/org/apache/datasketches/hll/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/hll/PreambleUtil.java
@@ -137,6 +137,7 @@ final class PreambleUtil {
   static final int EMPTY_FLAG_MASK          = 4;
   static final int COMPACT_FLAG_MASK        = 8;
   static final int OUT_OF_ORDER_FLAG_MASK   = 16;
+  static final int REBUILD_CURMIN_NUM_KXQ_MASK = 32; //used only by Union
 
   //Mode byte masks
   static final int CUR_MODE_MASK            = 3;
@@ -424,6 +425,18 @@ final class PreambleUtil {
     return (flags & OUT_OF_ORDER_FLAG_MASK) > 0;
   }
 
+  static void insertRebuildCurMinNumKxQFlag(final WritableMemory wmem, final boolean rebuild) {
+    int flags = wmem.getByte(FLAGS_BYTE);
+    if (rebuild) { flags |= REBUILD_CURMIN_NUM_KXQ_MASK; }
+    else { flags &= ~REBUILD_CURMIN_NUM_KXQ_MASK; }
+    wmem.putByte(FLAGS_BYTE, (byte) flags);
+  }
+
+  static boolean extractRebuildCurMinNumKxQFlag(final Memory mem) {
+    final int flags = mem.getByte(FLAGS_BYTE);
+    return (flags & REBUILD_CURMIN_NUM_KXQ_MASK) > 0;
+  }
+
   static void insertFlags(final WritableMemory wmem, final int flags) {
     wmem.putByte(FLAGS_BYTE, (byte) flags);
   }
diff --git a/src/main/java/org/apache/datasketches/hll/Union.java b/src/main/java/org/apache/datasketches/hll/Union.java
index 7127f33..cacde4b 100644
--- a/src/main/java/org/apache/datasketches/hll/Union.java
+++ b/src/main/java/org/apache/datasketches/hll/Union.java
@@ -19,6 +19,7 @@
 
 package org.apache.datasketches.hll;
 
+import static org.apache.datasketches.Util.invPow2;
 import static org.apache.datasketches.hll.HllUtil.EMPTY;
 import static org.apache.datasketches.hll.TgtHllType.HLL_8;
 
@@ -93,7 +94,10 @@ public class Union extends BaseHllSketch {
     lgMaxK = sketch.getLgConfigK();
     final TgtHllType tgtHllType = sketch.getTgtHllType();
     if (tgtHllType != TgtHllType.HLL_8) {
-      throw new SketchesArgumentException("Union can only wrap HLL_8 sketches.");
+      throw new SketchesArgumentException("Union can only wrap writable HLL_8 sketches.");
+    }
+    if (sketch.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(sketch.hllSketchImpl));
     }
     gadget = sketch;
   }
@@ -137,6 +141,9 @@ public class Union extends BaseHllSketch {
 
   @Override
   public double getCompositeEstimate() {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.hllSketchImpl.getCompositeEstimate();
   }
 
@@ -152,6 +159,9 @@ public class Union extends BaseHllSketch {
 
   @Override
   public double getEstimate() {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.getEstimate();
   }
 
@@ -178,6 +188,9 @@ public class Union extends BaseHllSketch {
 
   @Override
   public double getLowerBound(final int numStdDev) {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.getLowerBound(numStdDev);
   }
 
@@ -186,6 +199,9 @@ public class Union extends BaseHllSketch {
    * @return the result of this union operator as an HLL_4 sketch.
    */
   public HllSketch getResult() {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.copyAs(HllSketch.DEFAULT_HLL_TYPE);
   }
 
@@ -195,6 +211,9 @@ public class Union extends BaseHllSketch {
    * @return the result of this union operator with the specified TgtHllType
    */
   public HllSketch getResult(final TgtHllType tgtHllType) {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.copyAs(tgtHllType);
   }
 
@@ -210,6 +229,9 @@ public class Union extends BaseHllSketch {
 
   @Override
   public double getUpperBound(final int numStdDev) {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.getUpperBound(numStdDev);
   }
 
@@ -243,6 +265,14 @@ public class Union extends BaseHllSketch {
     return gadget.isSameResource(mem);
   }
 
+  boolean isRebuildCurMinNumKxQFlag() {
+    return gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag();
+  }
+
+  void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+    gadget.hllSketchImpl.putRebuildCurMinNumKxQFlag(rebuild);
+  }
+
   /**
    * Resets to empty and retains the current lgK, but does not change the configured value of
    * lgMaxK.
@@ -385,7 +415,8 @@ public class Union extends BaseHllSketch {
       case 4:  //src <= max, src >= gdt, gdtHLL, gdtHeap,   forward merge, no downsample, ooof=True
       {
         if ((srcLgK == gadgetLgK) && (source.getTgtHllType() == HLL_8) && (!source.isMemory())) {
-          specialMerge(source, gadget);
+          source.mergeTo(gadget);
+          //specialMerge(source, gadget);
         } else {
           source.mergeTo(gadget);    //merge src(Hll?,heap/mem,hll) -> gdt(Hll8,heap,hll), autofold
         }
@@ -427,7 +458,8 @@ public class Union extends BaseHllSketch {
       {
         final HllSketch gdtHll8Heap = downsample(gadget, srcLgK); //downsample gdt to srcLgK
         if ((source.getTgtHllType() == HLL_8) && (!source.isMemory())) {
-          specialMerge(source, gdtHll8Heap);
+          source.mergeTo(gdtHll8Heap);
+          //specialMerge(source, gdtHll8Heap);
         } else {
           source.mergeTo(gdtHll8Heap);//merge src(Hll?,heap/mem,hll) -> gdt(Hll8,heap,hll), autofold
         }
@@ -505,20 +537,17 @@ public class Union extends BaseHllSketch {
   /**
    * Source and target must both be type HLL_8, mode HLL, and with equal LgK.
    * @param source merge source
-   * @param target merge target
+   * @param target merge target, must be writable
    */
   private static void specialMerge(final HllSketch source, final HllSketch target) {
     final int k = 1 << target.getLgConfigK();
     final byte[] srcArr = ((Hll8Array) source.hllSketchImpl).hllByteArr;
     final Hll8Array tgtHll8Arr = (Hll8Array) target.hllSketchImpl;
     final byte[] tgtArr = tgtHll8Arr.hllByteArr;
-    int numZeros = 0;
     for (int i = 0; i < k; i++) {
-      final byte out = (srcArr[i] > tgtArr[i]) ? srcArr[i] : tgtArr[i];
-      tgtArr[i] = out;
-      if (out == 0) { numZeros++; }
+      tgtArr[i] = (srcArr[i] > tgtArr[i]) ? srcArr[i] : tgtArr[i];
     }
-    tgtHll8Arr.putNumAtCurMin(numZeros);
+    target.hllSketchImpl.putRebuildCurMinNumKxQFlag(true);
   }
 
   //Used by union operator.  Always copies or downsamples to Heap HLL_8.
@@ -543,4 +572,32 @@ public class Union extends BaseHllSketch {
     return new HllSketch(tgtHllArr);
   }
 
+  static final void curMinAndNumAndKxQ(final AbstractHllArray absHllArr) {
+    int curMin = 64;
+    int numAtCurMin = 0;
+    double kxq0 = 1 << absHllArr.getLgConfigK();
+    double kxq1 = 0;
+    final PairIterator itr = absHllArr.iterator();
+    while (itr.nextAll()) {
+      final int v = itr.getValue();
+      if (v > 0) {
+        if (v < 32) { kxq0 += invPow2(v) - 1.0; }
+        else        { kxq1 += invPow2(v) - 1.0; }
+      }
+      if (v > curMin) { continue; }
+      if (v < curMin) {
+        curMin = v;
+        numAtCurMin = 1;
+      } else {
+        numAtCurMin++;
+      }
+    }
+    absHllArr.putKxQ0(kxq0);
+    absHllArr.putKxQ1(kxq1);
+    absHllArr.putCurMin(curMin);
+    absHllArr.putNumAtCurMin(numAtCurMin);
+    absHllArr.putRebuildCurMinNumKxQFlag(false);
+    //HipAccum is not affected
+  }
+
 }
diff --git a/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java b/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
index 2cf24ac..ad76efb 100644
--- a/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
+++ b/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
@@ -24,10 +24,9 @@ import static org.apache.datasketches.hll.TgtHllType.HLL_6;
 import static org.apache.datasketches.hll.TgtHllType.HLL_8;
 import static org.testng.Assert.assertEquals;
 
-import org.testng.annotations.Test;
-
 import org.apache.datasketches.memory.Memory;
 import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
 
 /**
  * @author Lee Rhodes
@@ -77,6 +76,11 @@ public class ToFromByteArrayTest {
   }
 
   @Test
+  public void detailCheck() {
+    toFrom2(10, HLL_4, 24322); //24322 fails
+  }
+
+  @Test
   public void checkToFromSketch2() {
     for (int i = 0; i < 10; i++) {
       int n = nArr[i];
@@ -138,8 +142,8 @@ public class ToFromByteArrayTest {
   /**
    * @param s value to print
    */
-  static void println(String s) {
-    //System.out.println(s); //disable here
+  static void println(Object o) {
+    System.out.println(o.toString()); //disable here
   }
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 10/10: Merge branch 'master' into NewHllIterator

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit 838976bf79003984f5b6abcae52a73d45b833fb1
Merge: 1238051 34c0d93
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Dec 23 23:04:24 2019 -0800

    Merge branch 'master' into NewHllIterator

 src/main/java/org/apache/datasketches/hll/HllSketch.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 08/10: improve performance of integer and long byte conversions.

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit 72e91b4d514058ddeddf4f27fb88064edb5b9923
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Dec 23 16:55:38 2019 -0800

    improve performance of integer and long byte conversions.
---
 src/main/java/org/apache/datasketches/Util.java | 46 ++++++++++++++-----------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/Util.java b/src/main/java/org/apache/datasketches/Util.java
index 47891a1..4c8fe7f 100644
--- a/src/main/java/org/apache/datasketches/Util.java
+++ b/src/main/java/org/apache/datasketches/Util.java
@@ -133,11 +133,10 @@ public final class Util {
    * @return an int extracted from a Little-Endian byte array.
    */
   public static int bytesToInt(final byte[] arr) {
-    int v = 0;
-    for (int i = 0; i < 4; i++) {
-      v |= (arr[i] & 0XFF) << (i * 8);
-    }
-    return v;
+    return (((arr[3]       ) << 24)
+          | ((arr[2] & 0xff) << 16)
+          | ((arr[1] & 0xff) <<  8)
+          | ((arr[0] & 0xff)      ));
   }
 
   /**
@@ -146,11 +145,14 @@ public final class Util {
    * @return a long extracted from a Little-Endian byte array.
    */
   public static long bytesToLong(final byte[] arr) {
-    long v = 0;
-    for (int i = 0; i < 8; i++) {
-      v |= (arr[i] & 0XFFL) << (i * 8);
-    }
-    return v;
+    return ((((long)arr[7]       ) << 56)
+          | (((long)arr[6] & 0xff) << 48)
+          | (((long)arr[5] & 0xff) << 40)
+          | (((long)arr[4] & 0xff) << 32)
+          | (((long)arr[3] & 0xff) << 24)
+          | (((long)arr[2] & 0xff) << 16)
+          | (((long)arr[1] & 0xff) <<  8)
+          | (((long)arr[0] & 0xff)      ));
   }
 
   /**
@@ -159,11 +161,11 @@ public final class Util {
    * @param arr a given array of 4 bytes that will be returned with the data
    * @return a Little-Endian byte array extracted from the given int.
    */
-  public static byte[] intToBytes(int v, final byte[] arr) {
-    for (int i = 0; i < 4; i++) {
-      arr[i] = (byte) (v & 0XFF);
-      v >>>= 8;
-    }
+  public static byte[] intToBytes(final int v, final byte[] arr) {
+    arr[3] = (byte) (v >>> 24);
+    arr[2] = (byte) (v >>> 16);
+    arr[1] = (byte) (v >>>  8);
+    arr[0] = (byte) (v       );
     return arr;
   }
 
@@ -173,11 +175,15 @@ public final class Util {
    * @param arr a given array of 8 bytes that will be returned with the data
    * @return a Little-Endian byte array extracted from the given long.
    */
-  public static byte[] longToBytes(long v, final byte[] arr) {
-    for (int i = 0; i < 8; i++) {
-      arr[i] = (byte) (v & 0XFFL);
-      v >>>= 8;
-    }
+  public static byte[] longToBytes(final long v, final byte[] arr) {
+    arr[7] = (byte) (v >>> 56);
+    arr[6] = (byte) (v >>> 48);
+    arr[5] = (byte) (v >>> 40);
+    arr[4] = (byte) (v >>> 32);
+    arr[3] = (byte) (v >>> 24);
+    arr[2] = (byte) (v >>> 16);
+    arr[1] = (byte) (v >>>  8);
+    arr[0] = (byte) (v       );
     return arr;
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 03/10: restore to no checkReadOnly on DirectHllArray / putAuxHashMap().

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit b042a2e384304c48d81d8ffaf7cdca157be32b22
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Dec 19 12:16:28 2019 -0800

    restore to no checkReadOnly on DirectHllArray / putAuxHashMap().
---
 src/main/java/org/apache/datasketches/hll/DirectHllArray.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
index 60e7566..60ebdae 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java
@@ -189,7 +189,6 @@ abstract class DirectHllArray extends AbstractHllArray {
 
   @Override
   void putAuxHashMap(final AuxHashMap auxHashMap, final boolean compact) {
-    checkReadOnly(wmem);
     if (auxHashMap instanceof HeapAuxHashMap) {
       if (compact) {
         this.auxHashMap = auxHashMap; //heap and compact
@@ -202,7 +201,7 @@ abstract class DirectHllArray extends AbstractHllArray {
       }
     } else { //DirectAuxHashMap
       assert !compact; //must not be compact
-      this.auxHashMap = auxHashMap;
+      this.auxHashMap = auxHashMap; //In case of read-only this works.
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 05/10: Interim updates

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit e5b8a4263c4ba5f3b0c8e3183cc5d256e8999de4
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Dec 19 14:06:22 2019 -0800

    Interim updates
---
 .../apache/datasketches/hll/DirectCouponList.java  |  2 -
 .../java/org/apache/datasketches/hll/HllArray.java |  2 +
 .../apache/datasketches/hll/HllPairIterator.java   |  2 -
 .../org/apache/datasketches/hll/HllSketch.java     |  1 +
 .../org/apache/datasketches/hll/MiscTests.java     | 72 ----------------------
 5 files changed, 3 insertions(+), 76 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/DirectCouponList.java b/src/main/java/org/apache/datasketches/hll/DirectCouponList.java
index 52c8f93..26b5572 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectCouponList.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectCouponList.java
@@ -339,6 +339,4 @@ class DirectCouponList extends AbstractCoupons {
     return dirHllArr;
   }
 
-
-
 }
diff --git a/src/main/java/org/apache/datasketches/hll/HllArray.java b/src/main/java/org/apache/datasketches/hll/HllArray.java
index 37cbbe7..39ee5eb 100644
--- a/src/main/java/org/apache/datasketches/hll/HllArray.java
+++ b/src/main/java/org/apache/datasketches/hll/HllArray.java
@@ -28,6 +28,7 @@ import static org.apache.datasketches.hll.PreambleUtil.extractKxQ0;
 import static org.apache.datasketches.hll.PreambleUtil.extractKxQ1;
 import static org.apache.datasketches.hll.PreambleUtil.extractNumAtCurMin;
 import static org.apache.datasketches.hll.PreambleUtil.extractOooFlag;
+import static org.apache.datasketches.hll.PreambleUtil.extractRebuildCurMinNumKxQFlag;
 import static org.apache.datasketches.hll.TgtHllType.HLL_4;
 import static org.apache.datasketches.hll.TgtHllType.HLL_6;
 
@@ -255,6 +256,7 @@ abstract class HllArray extends AbstractHllArray {
     hllArray.putKxQ0(extractKxQ0(srcMem));
     hllArray.putKxQ1(extractKxQ1(srcMem));
     hllArray.putNumAtCurMin(extractNumAtCurMin(srcMem));
+    hllArray.putRebuildCurMinNumKxQFlag(extractRebuildCurMinNumKxQFlag(srcMem));
 
     //load Hll array
     srcMem.getByteArray(HLL_BYTE_ARR_START, hllArray.hllByteArr, 0, hllArray.hllByteArr.length);
diff --git a/src/main/java/org/apache/datasketches/hll/HllPairIterator.java b/src/main/java/org/apache/datasketches/hll/HllPairIterator.java
index b7d0c51..3d401a9 100644
--- a/src/main/java/org/apache/datasketches/hll/HllPairIterator.java
+++ b/src/main/java/org/apache/datasketches/hll/HllPairIterator.java
@@ -96,6 +96,4 @@ abstract class HllPairIterator extends PairIterator {
 
   abstract int value();
 
-
-
 }
diff --git a/src/main/java/org/apache/datasketches/hll/HllSketch.java b/src/main/java/org/apache/datasketches/hll/HllSketch.java
index ba1e222..e41ae0b 100644
--- a/src/main/java/org/apache/datasketches/hll/HllSketch.java
+++ b/src/main/java/org/apache/datasketches/hll/HllSketch.java
@@ -416,6 +416,7 @@ public class HllSketch extends BaseHllSketch {
         sb.append("  HipAccum       : ").append(absHll.getHipAccum()).append(LS);
         sb.append("  KxQ0           : ").append(absHll.getKxQ0()).append(LS);
         sb.append("  KxQ1           : ").append(absHll.getKxQ1()).append(LS);
+        sb.append("  Rebuild HIP Flg: ").append(absHll.isRebuildCurMinNumKxQFlag()).append(LS);
       } else {
         sb.append("  Coupon Count   : ")
           .append(((AbstractCoupons)hllSketchImpl).getCouponCount()).append(LS);
diff --git a/src/test/java/org/apache/datasketches/hll/MiscTests.java b/src/test/java/org/apache/datasketches/hll/MiscTests.java
deleted file mode 100644
index 0ded712..0000000
--- a/src/test/java/org/apache/datasketches/hll/MiscTests.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.hll;
-
-import org.testng.annotations.Test;
-
-/**
- * @author Lee Rhodes
- */
-@SuppressWarnings("javadoc")
-public class MiscTests {
-  long v = 0;
-
-  //@Test
-  public void checkSrcGTTgt() {
-    int unionLgK = 5;
-    int delLgK = 1;
-    int srcLgK = unionLgK + delLgK;
-    TgtHllType srcType = TgtHllType.HLL_8;
-
-    Union union = buildUnion(unionLgK, 1 << unionLgK);
-    HllSketch sk = build(srcLgK, srcType, 1 << srcLgK);
-    union.update(sk);
-
-  }
-
-
-
-
-  private Union buildUnion(int lgMaxK, int n) {
-    Union u = new Union(lgMaxK);
-    for (int i = 0; i < n; i++) { u.update(i + v); }
-    v += n;
-    return u;
-  }
-
-  private HllSketch build(int lgK, TgtHllType tgtHllType, int n) {
-    HllSketch sk = new HllSketch(lgK, tgtHllType);
-    for (int i = 0; i < n; i++) { sk.update(i + v); }
-    v += n;
-    return sk;
-  }
-
-  @Test
-  public void printlnTest() {
-    println("PRINTING: "+this.getClass().getName());
-  }
-
-  /**
-   * @param s value to print
-   */
-  static void println(String s) {
-    //System.out.println(s); //disable here
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 07/10: Added isomorphic tests.

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit d450b472f7f0bf39a35c9778cb4c29c0fed1f8bd
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Dec 23 15:23:12 2019 -0800

    Added isomorphic tests.
---
 .../java/org/apache/datasketches/hll/Union.java    |  39 ++--
 .../apache/datasketches/hll/IsomorphicTest.java    | 199 +++++++++++++++------
 2 files changed, 164 insertions(+), 74 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/Union.java b/src/main/java/org/apache/datasketches/hll/Union.java
index ae20156..d50dc1b 100644
--- a/src/main/java/org/apache/datasketches/hll/Union.java
+++ b/src/main/java/org/apache/datasketches/hll/Union.java
@@ -257,7 +257,7 @@ public class Union extends BaseHllSketch {
   }
 
   @Override
-  public boolean isOutOfOrderFlag() { //TODO return to pkg private
+  boolean isOutOfOrderFlag() {
     return gadget.isOutOfOrderFlag();
   }
 
@@ -331,28 +331,25 @@ public class Union extends BaseHllSketch {
   /**
    * Union the given source and destination sketches. This static method examines the state of
    * the current internal gadget and the incoming sketch and determines the optimum way to
-   * perform the union. This may involve swapping, down-sampling, transforming, and / or
+   * perform the union. This may involve swapping, downsampling, transforming, and / or
    * copying one of the arguments and may completely replace the internals of the union.
    *
-   * <p>A swap in update direction is required:</p>
-   * <ul><li>If the Gadget is in LIST, SET or EMPTY mode, AND
-   * The source sketch is in HLL mode.
-   * The source sketch will need to be copied or downsampled to the heap in HLL_8 mode if it is
-   * larger than maxLgK, and then replace the Gadget. If the Gadget was Memory, the resulting
-   * heap object will need to be converted to Memory form using the Gadget Memory.</li>
-   * </ul>
+   * <p>If the union gadget is empty, the source sketch is effectively copied to the union gadget
+   * after any required transformations.
    *
-   * <p>A downsample of gadget is required:</p>
-   * <ul><li>If both source and Gadget are in HLL mode AND source LgK <b>less than</b> Gadget LgK.
-   * Downsample Gadget, convert back to memory form if required, update gadget from source.</li>
-   * </ul>
+   * <p>The direction of the update is required if the union gadget is in LIST or SET mode, and the
+   * source sketch is in HLL mode. This is done to maintain maximum accuracy of the union process.
    *
-   * @param source the given incoming sketch, which must not be modified.
-   * @param gadget the given gadget sketch, which must have a target of HLL_8 and may be
-   * modified.
+   * <p>The source sketch is downsampled if the source LgK is larger than maxLgK and in HLL mode.
+   *
+   * <p>The union gadget is downsampled if both source and union gadget are in HLL mode
+   * and the source LgK <b>less than</b> the union gadget LgK.
+   *
+   * @param source the given incoming sketch, which is not be modified.
+   * @param gadget the given gadget sketch, which has a target of HLL_8 and may be modified.
    * @param lgMaxK the maximum value of log2 K for this union.
-   * @return the union of the two sketches in the form of the internal HllSketchImpl, which for
-   * the union is always in HLL_8 form.
+   * @return the union of the two sketches in the form of the internal HllSketchImpl, which is
+   * always in HLL_8 form.
    */
   private static HllSketchImpl unionImpl(final HllSketch source, final HllSketch gadget,
       final int lgMaxK) {
@@ -547,9 +544,9 @@ public class Union extends BaseHllSketch {
   }
 
   /**
-   * Source and target must both be type HLL_8, mode HLL, and with equal LgK.
-   * @param source merge source
-   * @param target merge target, must be writable
+   * Source and target must have the same LgK.
+   * @param source merge source on heap, type HLL_8, in HLL mode.
+   * @param target merge target. Must be writable, on heap, type HLL_8, in HLL mode.
    */
   private static void specialMerge(final HllSketch source, final HllSketch target) {
     final int k = 1 << target.getLgConfigK();
diff --git a/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java b/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
index a82ea71..8bb5fbb 100644
--- a/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
+++ b/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
@@ -19,8 +19,11 @@
 
 package org.apache.datasketches.hll;
 
+import static org.apache.datasketches.hll.CurMode.HLL;
 import static org.apache.datasketches.hll.CurMode.LIST;
 import static org.apache.datasketches.hll.CurMode.SET;
+import static org.apache.datasketches.hll.TgtHllType.HLL_8;
+import static org.testng.Assert.fail;
 
 import org.apache.datasketches.memory.WritableMemory;
 import org.testng.annotations.Test;
@@ -36,68 +39,69 @@ public class IsomorphicTest {
   Union union;
 
   @Test
+  //Merges a type1 to an empty union (heap, HLL_8), and gets result as type1, checks binary equivalence
   public void isomorphsUnionUpdatableHeap() {
-    for (int lgK = 4; lgK <= 21; lgK++) {
-      for (int cm = 0; cm <= 2; cm++) {
+    for (int lgK = 4; lgK <= 21; lgK++) { //All LgK
+      for (int cm = 0; cm <= 2; cm++) { //List, Set, Hll
         if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
         CurMode curMode = CurMode.fromOrdinal(cm);
-        for (int t = 0; t <= 2; t++) {
-          TgtHllType tgtHllType = TgtHllType.fromOrdinal(t);
-          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType, curMode);
-          byte[] sk1bytes = sk1.toUpdatableByteArray();
-          Union union = buildHeapUnion(lgK, null);
+        for (int t = 0; t <= 2; t++) { //HLL_4, HLL_6, HLL_8
+          TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
+          byte[] sk1bytes = sk1.toUpdatableByteArray(); //UPDATABLE
+          Union union = buildHeapUnion(lgK, null); //UNION
           union.update(sk1);
-          HllSketch sk2 = union.getResult(tgtHllType);
-          byte[] sk2bytes = sk2.toUpdatableByteArray();
-          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType;
-          checkArrays(sk1bytes, sk2bytes, comb);
+          HllSketch sk2 = union.getResult(tgtHllType1);
+          byte[] sk2bytes = sk2.toUpdatableByteArray(); //UPDATABLE
+          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType1;
+          checkArrays(sk1bytes, sk2bytes, comb, false);
         }
       }
     }
   }
 
   @Test
+  //Merges a type1 to an empty union (heap, HLL_8), and gets result as type1, checks binary equivalence
   public void isomorphsUnionCompactHeap() {
-    for (int lgK = 4; lgK <= 21; lgK++) {
-      for (int cm = 0; cm <= 2; cm++) {
+    for (int lgK = 4; lgK <= 21; lgK++) { //All LgK
+      for (int cm = 0; cm <= 2; cm++) { //List, Set, Hll
         if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
         CurMode curMode = CurMode.fromOrdinal(cm);
-        for (int t = 0; t <= 2; t++) {
-          TgtHllType tgtHllType = TgtHllType.fromOrdinal(t);
-          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType, curMode);
-          byte[] sk1bytes = sk1.toCompactByteArray();
-          Union union = buildHeapUnion(lgK, null);
+        for (int t = 0; t <= 2; t++) { //HLL_4, HLL_6, HLL_8
+          TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
+          byte[] sk1bytes = sk1.toCompactByteArray(); //COMPACT
+          Union union = buildHeapUnion(lgK, null); //UNION
           union.update(sk1);
-          HllSketch sk2 = union.getResult(tgtHllType);
-          byte[] sk2bytes = sk2.toCompactByteArray();
-          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType;
-          checkArrays(sk1bytes, sk2bytes, comb);
+          HllSketch sk2 = union.getResult(tgtHllType1);
+          byte[] sk2bytes = sk2.toCompactByteArray(); //COMPACT
+          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType1;
+          checkArrays(sk1bytes, sk2bytes, comb, false);
         }
       }
     }
   }
 
-
   @Test
+  //Converts a type1 to a different type and converts back to type1 to check binary equivalence.
   public void isomorphsCopyAsUpdatableHeap() {
-    for (int lgK = 4; lgK <= 21; lgK++) {
-      for (int cm = 0; cm <= 2; cm++) {
+    for (int lgK = 4; lgK <= 21; lgK++) { //All LgK
+      for (int cm = 0; cm <= 2; cm++) { //List, Set, Hll
         if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
         CurMode curMode = CurMode.fromOrdinal(cm);
-        for (int t1 = 0; t1 <= 2; t1++) {
+        for (int t1 = 0; t1 <= 2; t1++) { //HLL_4, HLL_6, HLL_8
           TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t1);
           HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
-          byte[] sk1bytes = sk1.toUpdatableByteArray();
-
-          for (int t2 = 0; t2 <= 2; t2++) {
+          byte[] sk1bytes = sk1.toUpdatableByteArray(); //UPDATABLE
+          for (int t2 = 0; t2 <= 2; t2++) { //HLL_4, HLL_6, HLL_8
             if (t2 == t1) { continue; }
             TgtHllType tgtHllType2 = TgtHllType.fromOrdinal(t2);
-            HllSketch sk2 = sk1.copyAs(tgtHllType2);
-            HllSketch sk3 = sk2.copyAs(tgtHllType1);
-            byte[] sk3bytes = sk3.toUpdatableByteArray();
+            HllSketch sk2 = sk1.copyAs(tgtHllType2); //COPY AS
+            HllSketch sk1B = sk2.copyAs(tgtHllType1); //COPY AS
+            byte[] sk1Bbytes = sk1B.toUpdatableByteArray(); //UPDATABLE
             String comb = "LgK= " + lgK + ", CurMode= " + curMode.toString()
             + ", Type1: " + tgtHllType1 + ", Type2: " + tgtHllType2;
-            checkArrays(sk1bytes, sk3bytes, comb);
+            checkArrays(sk1bytes, sk1Bbytes, comb, false);
           }
         }
       }
@@ -105,49 +109,98 @@ public class IsomorphicTest {
   }
 
   @Test
+  //Converts a type1 to a different type and converts back to type1 to check binary equivalence.
   public void isomorphsCopyAsCompactHeap() {
-    for (int lgK = 4; lgK <= 21; lgK++) {
-      for (int cm = 0; cm <= 2; cm++) {
+    for (int lgK = 4; lgK <= 21; lgK++) { //All LgK
+      for (int cm = 0; cm <= 2; cm++) { //List, Set, Hll
         if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
         CurMode curMode = CurMode.fromOrdinal(cm);
-        for (int t1 = 0; t1 <= 2; t1++) {
+        for (int t1 = 0; t1 <= 2; t1++) { //HLL_4, HLL_6, HLL_8
           TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t1);
           HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
-          byte[] sk1bytes = sk1.toCompactByteArray();
-
-          for (int t2 = 0; t2 <= 2; t2++) {
+          byte[] sk1bytes = sk1.toCompactByteArray(); //COMPACT
+          for (int t2 = 0; t2 <= 2; t2++) { //HLL_4, HLL_6, HLL_8
             if (t2 == t1) { continue; }
             TgtHllType tgtHllType2 = TgtHllType.fromOrdinal(t2);
-            HllSketch sk2 = sk1.copyAs(tgtHllType2);
-            HllSketch sk3 = sk2.copyAs(tgtHllType1);
-            byte[] sk3bytes = sk3.toCompactByteArray();
+            HllSketch sk2 = sk1.copyAs(tgtHllType2); //COPY AS
+            HllSketch sk3 = sk2.copyAs(tgtHllType1); //COPY AS
+            byte[] sk3bytes = sk3.toCompactByteArray(); //COMPACT
             String comb = "LgK= " + lgK + ", CurMode= " + curMode.toString()
             + ", Type1: " + tgtHllType1 + ", Type2: " + tgtHllType2;
-            checkArrays(sk1bytes, sk3bytes, comb);
+            checkArrays(sk1bytes, sk3bytes, comb, false);
           }
         }
       }
     }
   }
 
+  @Test
+  //Compares two HLL to HLL merges, combinations of heap, memory for binary equivalence.
+  public void isomorphsHllMerges() {
+    for (int lgK = 4; lgK <= 21; lgK++) { //All LgK
+      for (int t1 = 0; t1 <= 2; t1++) { //HLL_4, HLL_6, HLL_8
+        TgtHllType tgtHllType = TgtHllType.fromOrdinal(t1);
+        Union u;
+        HllSketch sk, skOut;
+
+        //CASE 1 Heap Union, Heap sketch
+        // (this uses the special merge, so HipAccum (8 bytes) will differ for HLL8)
+        u = buildHeapUnionHllMode(lgK, 0);
+        sk = buildHeapSketchHllMode(lgK, tgtHllType, 1 << lgK);
+        u.update(sk);
+        byte[] bytesOut1 = u.getResult(HLL_8).toUpdatableByteArray();
+
+        //CASE 2 Heap Union, Memory sketch
+        u = buildHeapUnionHllMode(lgK, 0);
+        sk = buildMemorySketchHllMode(lgK, tgtHllType, 1 << lgK);
+        u.update(sk);
+        byte[] bytesOut2 = u.getResult(HLL_8).toUpdatableByteArray();
+
+        //println("Uheap/SkHeap   HIP: " + bytesToDouble(bytesOut1, 8)); //HipAccum
+        //println("Uheap/SkMemory HIP: " + bytesToDouble(bytesOut2, 8)); //HipAccum
+        String comb = "LgK: " + lgK + ", SkType: " + tgtHllType.toString()
+        + ", Case1: Heap Union, Heap sketch; Case2: /Heap Union, Memory sketch";
+        checkArrays(bytesOut1, bytesOut2, comb, true);
+
+        //CASE 3 Offheap Union, Heap sketch
+        u = buildMemoryUnionHllMode(lgK, 0);
+        sk = buildHeapSketchHllMode(lgK, tgtHllType, 1 << lgK);
+        u.update(sk);
+        byte[] bytesOut3 = u.getResult(HLL_8).toUpdatableByteArray();
+
+        //println("Uheap/SkMemory HIP: " + bytesToDouble(bytesOut2, 8)); //HipAccum
+        //println("Umemory/SkHeap HIP: " + bytesToDouble(bytesOut3, 8)); //HipAccum
+        comb = "LgK: " + lgK + ", SkType: " + tgtHllType.toString()
+        + ", Case2: Heap Union, Memory sketch; Case3: /Memory Union, Heap sketch";
+        checkArrays(bytesOut2, bytesOut3, comb, false);
 
-  private static void checkArrays(byte[] sk1, byte[] sk2, String comb) {
+        //Case 4 Memory Union, Memory sketch
+        u = buildMemoryUnionHllMode(lgK, 0);
+        sk = buildMemorySketchHllMode(lgK, tgtHllType, 1 << lgK);
+        u.update(sk);
+        byte[] bytesOut4 = u.getResult(HLL_8).toUpdatableByteArray();
+
+        comb = "LgK: " + lgK + ", SkType: " + tgtHllType.toString()
+        + ", Case2: Heap Union, Memory sketch; Case4: /Memory Union, Memory sketch";
+        checkArrays(bytesOut2, bytesOut4, comb, false);
+      }
+    }
+  }
+
+  private static void checkArrays(byte[] sk1, byte[] sk2, String comb, boolean omitHipAccum) {
     int len = sk1.length;
     if (len != sk2.length) {
       println("Sketch images not the same length: " + comb);
       return;
     }
-
-    boolean first = true;
+    print(comb + ": ");
     for (int i = 0; i < len; i++) {
+      if (omitHipAccum && (i >= 8) && (i <= 15)) { continue; }
       if (sk1[i] == sk2[i]) { continue; }
-      if (first) {
-        print(comb + ": ");
-        first = false;
-      }
       print(i + " ");
+      fail();
     }
-    if (!first) { println(""); }
+    println("");
   }
 
 
@@ -188,11 +241,51 @@ public class IsomorphicTest {
     return sk;
   }
 
-  //lgK >= 8
+  private static Union buildHeapUnionHllMode(int lgMaxK, int startN) {
+    Union u = new Union(lgMaxK);
+    int n = getN(lgMaxK, HLL);
+    for (int i = 0; i < n; i++) { u.update(i + startN); }
+    return u;
+  }
+
+  private static Union buildMemoryUnionHllMode(int lgMaxK, int startN) {
+    final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgMaxK, TgtHllType.HLL_8);
+    WritableMemory wmem = WritableMemory.allocate(bytes);
+    Union u = new Union(lgMaxK, wmem);
+    int n = getN(lgMaxK, HLL);
+    for (int i = 0; i < n; i++) { u.update(i + startN); }
+    return u;
+  }
+
+  private static HllSketch buildHeapSketchHllMode(int lgK, TgtHllType tgtHllType, int startN) {
+    HllSketch sk = new HllSketch(lgK, tgtHllType);
+    int n = getN(lgK, HLL);
+    for (int i = 0; i < n; i++) { sk.update(i + startN); }
+    return sk;
+  }
+
+  private static HllSketch buildMemorySketchHllMode(int lgK, TgtHllType tgtHllType, int startN) {
+    final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK,tgtHllType);
+    WritableMemory wmem = WritableMemory.allocate(bytes);
+    HllSketch sk = new HllSketch(lgK, tgtHllType, wmem);
+    int n = getN(lgK, HLL);
+    for (int i = 0; i < n; i++) { sk.update(i + startN); }
+    return sk;
+  }
+
+  //if lgK >= 8, curMode != SET!
   private static int getN(int lgK, CurMode curMode) {
     if (curMode == LIST) { return 4; }
     if (curMode == SET) { return 1 << (lgK - 4); }
-    return 1 << (lgK - 3); //HLL
+    return ((lgK < 8) && (curMode == HLL)) ? (1 << lgK) : 1 << (lgK - 3);
+  }
+
+  private static double bytesToDouble(byte[] arr, int offset) {
+    long v = 0;
+    for (int i = offset; i < (offset + 8); i++) {
+      v |= (arr[i] & 0XFFL) << (i * 8);
+    }
+    return Double.longBitsToDouble(v);
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 06/10: Added Isomorphism tests, and a few others.

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit 93e4ca2bccc305cf230da0eaa9bbfb11e05af4fd
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Fri Dec 20 21:25:21 2019 -0800

    Added Isomorphism tests, and a few others.
---
 .../apache/datasketches/hll/ToByteArrayImpl.java   |   2 +
 .../java/org/apache/datasketches/hll/Union.java    |  33 ++-
 .../apache/datasketches/hll/CouponListTest.java    |   9 +-
 .../org/apache/datasketches/hll/HllArrayTest.java  |  10 +-
 .../apache/datasketches/hll/IsomorphicTest.java    | 224 +++++++++++++++++++++
 .../org/apache/datasketches/hll/UnionCaseTest.java |  51 ++++-
 6 files changed, 311 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/ToByteArrayImpl.java b/src/main/java/org/apache/datasketches/hll/ToByteArrayImpl.java
index 2705fe7..2179ff7 100644
--- a/src/main/java/org/apache/datasketches/hll/ToByteArrayImpl.java
+++ b/src/main/java/org/apache/datasketches/hll/ToByteArrayImpl.java
@@ -40,6 +40,7 @@ import static org.apache.datasketches.hll.PreambleUtil.insertListCount;
 import static org.apache.datasketches.hll.PreambleUtil.insertNumAtCurMin;
 import static org.apache.datasketches.hll.PreambleUtil.insertOooFlag;
 import static org.apache.datasketches.hll.PreambleUtil.insertPreInts;
+import static org.apache.datasketches.hll.PreambleUtil.insertRebuildCurMinNumKxQFlag;
 import static org.apache.datasketches.hll.PreambleUtil.insertSerVer;
 import static org.apache.datasketches.hll.PreambleUtil.insertTgtHllType;
 
@@ -100,6 +101,7 @@ class ToByteArrayImpl {
     insertKxQ0(tgtWmem, srcImpl.getKxQ0());
     insertKxQ1(tgtWmem, srcImpl.getKxQ1());
     insertNumAtCurMin(tgtWmem, srcImpl.getNumAtCurMin());
+    insertRebuildCurMinNumKxQFlag(tgtWmem, srcImpl.isRebuildCurMinNumKxQFlag());
   }
 
   private static final void insertAux(final AbstractHllArray srcImpl, final WritableMemory tgtWmem,
diff --git a/src/main/java/org/apache/datasketches/hll/Union.java b/src/main/java/org/apache/datasketches/hll/Union.java
index 17878e9..ae20156 100644
--- a/src/main/java/org/apache/datasketches/hll/Union.java
+++ b/src/main/java/org/apache/datasketches/hll/Union.java
@@ -96,8 +96,9 @@ public class Union extends BaseHllSketch {
     if (tgtHllType != TgtHllType.HLL_8) {
       throw new SketchesArgumentException("Union can only wrap writable HLL_8 sketches.");
     }
+    //This should never happen
     if (sketch.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(sketch.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(sketch.hllSketchImpl));
     }
     gadget = sketch;
   }
@@ -142,7 +143,7 @@ public class Union extends BaseHllSketch {
   @Override
   public double getCompositeEstimate() {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.hllSketchImpl.getCompositeEstimate();
   }
@@ -160,7 +161,7 @@ public class Union extends BaseHllSketch {
   @Override
   public double getEstimate() {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.getEstimate();
   }
@@ -189,7 +190,7 @@ public class Union extends BaseHllSketch {
   @Override
   public double getLowerBound(final int numStdDev) {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.getLowerBound(numStdDev);
   }
@@ -200,7 +201,7 @@ public class Union extends BaseHllSketch {
    */
   public HllSketch getResult() {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.copyAs(HllSketch.DEFAULT_HLL_TYPE);
   }
@@ -212,7 +213,7 @@ public class Union extends BaseHllSketch {
    */
   public HllSketch getResult(final TgtHllType tgtHllType) {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.copyAs(tgtHllType);
   }
@@ -230,7 +231,7 @@ public class Union extends BaseHllSketch {
   @Override
   public double getUpperBound(final int numStdDev) {
     if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
-      curMinAndNumAndKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
     }
     return gadget.getUpperBound(numStdDev);
   }
@@ -291,11 +292,17 @@ public class Union extends BaseHllSketch {
    */
   @Override
   public byte[] toCompactByteArray() {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.toCompactByteArray();
   }
 
   @Override
   public byte[] toUpdatableByteArray() {
+    if (gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag()) {
+      rebuildCurMinNumKxQ((AbstractHllArray)(gadget.hllSketchImpl));
+    }
     return gadget.toUpdatableByteArray();
   }
 
@@ -351,19 +358,24 @@ public class Union extends BaseHllSketch {
       final int lgMaxK) {
     assert gadget.getTgtHllType() == HLL_8;
     if ((source == null) || source.isEmpty()) { return gadget.hllSketchImpl; }
+    final int srcLgK = source.getLgConfigK();
+    final int gadgetLgK = gadget.getLgConfigK();
+    //Source is either LIST or SET mode
     if (source.getCurMode() == CurMode.LIST ) {
       source.mergeTo(gadget);
       gadget.putOutOfOrderFlag(gadget.isOutOfOrderFlag() | source.isOutOfOrderFlag());
       return gadget.hllSketchImpl;
     }
     if (source.getCurMode() == CurMode.SET ) {
+      if (gadget.isEmpty() && (!source.isMemory()) && (!gadget.isMemory()) && (srcLgK == gadgetLgK)) {
+        gadget.hllSketchImpl = source.copyAs(HLL_8).hllSketchImpl;
+        return gadget.hllSketchImpl;
+      }
       source.mergeTo(gadget);
       gadget.putOutOfOrderFlag(true);
       return gadget.hllSketchImpl;
     }
     //Hereafter, the srcImpl is in HLL mode.
-    final int srcLgK = source.getLgConfigK();
-    final int gadgetLgK = gadget.getLgConfigK();
     final int bit0 = gadget.isMemory() ? 1 : 0;
     final int bits1_2 = (gadget.isEmpty() ? 3 : gadget.getCurMode().ordinal()) << 1;
     final int bit3 = (srcLgK < gadgetLgK) ? 8 : 0;
@@ -572,7 +584,8 @@ public class Union extends BaseHllSketch {
     return new HllSketch(tgtHllArr);
   }
 
-  static final void curMinAndNumAndKxQ(final AbstractHllArray absHllArr) {
+  //
+  private static final void rebuildCurMinNumKxQ(final AbstractHllArray absHllArr) {
     int curMin = 64;
     int numAtCurMin = 0;
     double kxq0 = 1 << absHllArr.getLgConfigK();
diff --git a/src/test/java/org/apache/datasketches/hll/CouponListTest.java b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
index fc9eda1..3f81f88 100644
--- a/src/test/java/org/apache/datasketches/hll/CouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
@@ -24,9 +24,8 @@ import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
-import org.testng.annotations.Test;
-
 import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
 
 /**
  * @author Lee Rhodes
@@ -77,6 +76,9 @@ public class CouponListTest {
     }
     assertEquals(sk.getCurMode(), CurMode.LIST);
     assertEquals(sk.getCompositeEstimate(), 7.0, 7 * .01);
+    assertEquals(sk.getHipEstimate(), 7.0, 7 * .01);
+    sk.hllSketchImpl.putEmptyFlag(false); //dummy
+    sk.hllSketchImpl.putRebuildCurMinNumKxQFlag(false); //dummy
     if (direct) {
       assertNotNull(sk.hllSketchImpl.getWritableMemory());
     } else {
@@ -87,6 +89,7 @@ public class CouponListTest {
     sk.update(8);
     assertEquals(sk.getCurMode(), CurMode.SET);
     assertEquals(sk.getCompositeEstimate(), 8.0, 8 * .01);
+    assertEquals(sk.getHipEstimate(), 8.0, 8 * .01);
     if (direct) {
       assertNotNull(sk.hllSketchImpl.getWritableMemory());
     } else {
@@ -107,6 +110,8 @@ public class CouponListTest {
 
     double re = sk.getRelErr(true, true, 4, 1);
     assertTrue(re < 0.0);
+
+
   }
 
   @Test
diff --git a/src/test/java/org/apache/datasketches/hll/HllArrayTest.java b/src/test/java/org/apache/datasketches/hll/HllArrayTest.java
index d9180a8..f198b4a 100644
--- a/src/test/java/org/apache/datasketches/hll/HllArrayTest.java
+++ b/src/test/java/org/apache/datasketches/hll/HllArrayTest.java
@@ -99,7 +99,9 @@ public class HllArrayTest {
     if (sk1.hllSketchImpl instanceof HllArray) {
       assertFalse(sk1.hllSketchImpl.isMemory());
       assertFalse(sk1.isSameResource(wmem));
-    } else {
+      int v = ((AbstractHllArray)sk1.hllSketchImpl).getSlot(0);
+      ((AbstractHllArray)sk1.hllSketchImpl).putSlot(0, v);
+    } else { //DirectHllArray
       assertTrue(sk1.hllSketchImpl.isMemory());
       assertTrue(sk1.isSameResource(wmem));
     }
@@ -107,6 +109,12 @@ public class HllArrayTest {
     //sk1.update(u);
     double est1 = sk1.getEstimate();
     assertEquals(est1, u, u * .03);
+    assertEquals(sk1.getHipEstimate(), est1, 0.0);
+
+    //misc calls
+    sk1.hllSketchImpl.putEmptyFlag(false);
+    sk1.hllSketchImpl.putRebuildCurMinNumKxQFlag(true);
+    sk1.hllSketchImpl.putRebuildCurMinNumKxQFlag(false);
 
     byte[] byteArray = sk1.toCompactByteArray();
     HllSketch sk2 = HllSketch.heapify(byteArray);
diff --git a/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java b/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
new file mode 100644
index 0000000..a82ea71
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll/IsomorphicTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll;
+
+import static org.apache.datasketches.hll.CurMode.LIST;
+import static org.apache.datasketches.hll.CurMode.SET;
+
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+@SuppressWarnings({"javadoc", "unused"})
+public class IsomorphicTest {
+  long v = 0;
+  final int maxLgK = 12;
+  HllSketch source;
+  Union union;
+
+  @Test
+  public void isomorphsUnionUpdatableHeap() {
+    for (int lgK = 4; lgK <= 21; lgK++) {
+      for (int cm = 0; cm <= 2; cm++) {
+        if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
+        CurMode curMode = CurMode.fromOrdinal(cm);
+        for (int t = 0; t <= 2; t++) {
+          TgtHllType tgtHllType = TgtHllType.fromOrdinal(t);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType, curMode);
+          byte[] sk1bytes = sk1.toUpdatableByteArray();
+          Union union = buildHeapUnion(lgK, null);
+          union.update(sk1);
+          HllSketch sk2 = union.getResult(tgtHllType);
+          byte[] sk2bytes = sk2.toUpdatableByteArray();
+          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType;
+          checkArrays(sk1bytes, sk2bytes, comb);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void isomorphsUnionCompactHeap() {
+    for (int lgK = 4; lgK <= 21; lgK++) {
+      for (int cm = 0; cm <= 2; cm++) {
+        if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
+        CurMode curMode = CurMode.fromOrdinal(cm);
+        for (int t = 0; t <= 2; t++) {
+          TgtHllType tgtHllType = TgtHllType.fromOrdinal(t);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType, curMode);
+          byte[] sk1bytes = sk1.toCompactByteArray();
+          Union union = buildHeapUnion(lgK, null);
+          union.update(sk1);
+          HllSketch sk2 = union.getResult(tgtHllType);
+          byte[] sk2bytes = sk2.toCompactByteArray();
+          String comb = "LgK=" + lgK + ", CurMode=" + curMode.toString() + ", Type:" + tgtHllType;
+          checkArrays(sk1bytes, sk2bytes, comb);
+        }
+      }
+    }
+  }
+
+
+  @Test
+  public void isomorphsCopyAsUpdatableHeap() {
+    for (int lgK = 4; lgK <= 21; lgK++) {
+      for (int cm = 0; cm <= 2; cm++) {
+        if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
+        CurMode curMode = CurMode.fromOrdinal(cm);
+        for (int t1 = 0; t1 <= 2; t1++) {
+          TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t1);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
+          byte[] sk1bytes = sk1.toUpdatableByteArray();
+
+          for (int t2 = 0; t2 <= 2; t2++) {
+            if (t2 == t1) { continue; }
+            TgtHllType tgtHllType2 = TgtHllType.fromOrdinal(t2);
+            HllSketch sk2 = sk1.copyAs(tgtHllType2);
+            HllSketch sk3 = sk2.copyAs(tgtHllType1);
+            byte[] sk3bytes = sk3.toUpdatableByteArray();
+            String comb = "LgK= " + lgK + ", CurMode= " + curMode.toString()
+            + ", Type1: " + tgtHllType1 + ", Type2: " + tgtHllType2;
+            checkArrays(sk1bytes, sk3bytes, comb);
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  public void isomorphsCopyAsCompactHeap() {
+    for (int lgK = 4; lgK <= 21; lgK++) {
+      for (int cm = 0; cm <= 2; cm++) {
+        if ((lgK < 8) && (cm == 1)) { continue; } //lgk < 8 list transistions directly to HLL
+        CurMode curMode = CurMode.fromOrdinal(cm);
+        for (int t1 = 0; t1 <= 2; t1++) {
+          TgtHllType tgtHllType1 = TgtHllType.fromOrdinal(t1);
+          HllSketch sk1 = buildHeapSketch(lgK, tgtHllType1, curMode);
+          byte[] sk1bytes = sk1.toCompactByteArray();
+
+          for (int t2 = 0; t2 <= 2; t2++) {
+            if (t2 == t1) { continue; }
+            TgtHllType tgtHllType2 = TgtHllType.fromOrdinal(t2);
+            HllSketch sk2 = sk1.copyAs(tgtHllType2);
+            HllSketch sk3 = sk2.copyAs(tgtHllType1);
+            byte[] sk3bytes = sk3.toCompactByteArray();
+            String comb = "LgK= " + lgK + ", CurMode= " + curMode.toString()
+            + ", Type1: " + tgtHllType1 + ", Type2: " + tgtHllType2;
+            checkArrays(sk1bytes, sk3bytes, comb);
+          }
+        }
+      }
+    }
+  }
+
+
+  private static void checkArrays(byte[] sk1, byte[] sk2, String comb) {
+    int len = sk1.length;
+    if (len != sk2.length) {
+      println("Sketch images not the same length: " + comb);
+      return;
+    }
+
+    boolean first = true;
+    for (int i = 0; i < len; i++) {
+      if (sk1[i] == sk2[i]) { continue; }
+      if (first) {
+        print(comb + ": ");
+        first = false;
+      }
+      print(i + " ");
+    }
+    if (!first) { println(""); }
+  }
+
+
+  //BUILDERS
+  private Union buildHeapUnion(int lgMaxK, CurMode curMode) {
+    Union u = new Union(lgMaxK);
+    int n = (curMode == null) ? 0 : getN(lgMaxK, curMode);
+    for (int i = 0; i < n; i++) { u.update(i + v); }
+    v += n;
+    return u;
+  }
+
+  private Union buildMemoryUnion(int lgMaxK, CurMode curMode) {
+    final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgMaxK, TgtHllType.HLL_8);
+    WritableMemory wmem = WritableMemory.allocate(bytes);
+    Union u = new Union(lgMaxK, wmem);
+    int n = (curMode == null) ? 0 : getN(lgMaxK, curMode);
+    for (int i = 0; i < n; i++) { u.update(i + v); }
+    v += n;
+    return u;
+  }
+
+  private HllSketch buildHeapSketch(int lgK, TgtHllType tgtHllType, CurMode curMode) {
+    HllSketch sk = new HllSketch(lgK, tgtHllType);
+    int n = (curMode == null) ? 0 : getN(lgK, curMode);
+    for (int i = 0; i < n; i++) { sk.update(i + v); }
+    v += n;
+    return sk;
+  }
+
+  private HllSketch buildMemorySketch(int lgK, TgtHllType tgtHllType, CurMode curMode) {
+    final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK,tgtHllType);
+    WritableMemory wmem = WritableMemory.allocate(bytes);
+    HllSketch sk = new HllSketch(lgK, tgtHllType, wmem);
+    int n = (curMode == null) ? 0 : getN(lgK, curMode);
+    for (int i = 0; i < n; i++) { sk.update(i + v); }
+    v += n;
+    return sk;
+  }
+
+  //lgK >= 8
+  private static int getN(int lgK, CurMode curMode) {
+    if (curMode == LIST) { return 4; }
+    if (curMode == SET) { return 1 << (lgK - 4); }
+    return 1 << (lgK - 3); //HLL
+  }
+
+  @Test
+  public void printlnTest() {
+    println("PRINTING: "+this.getClass().getName());
+  }
+
+  /**
+   * @param o value to print
+   */
+  static void println(Object o) {
+    print(o.toString() + "\n");
+  }
+
+  /**
+   * @param o value to print
+   */
+  static void print(Object o) {
+    //System.out.print(o.toString()); //disable here
+  }
+
+  /**
+   * @param fmt format
+   * @param args arguments
+   */
+  static void printf(String fmt, Object...args) {
+    //System.out.printf(fmt, args); //disable here
+  }
+}
diff --git a/src/test/java/org/apache/datasketches/hll/UnionCaseTest.java b/src/test/java/org/apache/datasketches/hll/UnionCaseTest.java
index 3272295..2459850 100644
--- a/src/test/java/org/apache/datasketches/hll/UnionCaseTest.java
+++ b/src/test/java/org/apache/datasketches/hll/UnionCaseTest.java
@@ -119,7 +119,6 @@ public class UnionCaseTest {
     assertTrue(err < rse);
   }
 
-
   private HllSketch getSource(int caseNum, TgtHllType tgtHllType, boolean memory) {
     int srcLgK = getSrcLgK(caseNum, maxLgK);
     int srcU = getSrcCount(caseNum, maxLgK);
@@ -197,7 +196,7 @@ public class UnionCaseTest {
   }
 
   @Test
-  public void checkSrcListSet() { //src: SET, gadget: LIST, cases 0, 1
+  public void checkSrcListSet() { //src: SET, gadget: LIST
     int n1 = 5;
     int n2 = 2;
     int n3 = 16;
@@ -219,7 +218,7 @@ public class UnionCaseTest {
   }
 
   @Test
-  public void checkSrcSetList() { //src: LIST, gadget: SET, cases 0, 4
+  public void checkSrcSetList() { //src: LIST, gadget: SET
     int n1 = 6;
     int n2 = 10;
     int n3 = 6;
@@ -241,7 +240,7 @@ public class UnionCaseTest {
   }
 
   @Test
-  public void checkSrcSetSet() { //src: SET, gadget: SET, cases 0, 5
+  public void checkSrcSetSet() { //src: SET, gadget: SET
     int n1 = 6;
     int n2 = 10;
     int n3 = 16;
@@ -263,7 +262,7 @@ public class UnionCaseTest {
   }
 
   @Test
-  public void checkSrcEmptyList() { //src: LIST, gadget: empty, case 12
+  public void checkSrcEmptyList() { //src: LIST, gadget: empty
     int n1 = 0;
     int n2 = 0;
     int n3 = 7;
@@ -306,6 +305,48 @@ public class UnionCaseTest {
     assertEquals(u.getEstimate(), sum, err);
   }
 
+  @SuppressWarnings("unused")
+  @Test
+  public void checkSpecialMergeCase4() {
+    Union u = buildHeapUnion(12, 1 << 9);
+    HllSketch sk = buildHeapSketch(12, HLL_8, 1 << 9);
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    u.getCompositeEstimate();
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    u.getLowerBound(2);
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    u.getUpperBound(2);
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    u.getResult();
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    byte[] ba = u.toCompactByteArray();
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.update(sk);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    ba = u.toUpdatableByteArray();
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+
+    u.putRebuildCurMinNumKxQFlag(true);
+    assertTrue(u.isRebuildCurMinNumKxQFlag());
+    u.putRebuildCurMinNumKxQFlag(false);
+    assertFalse(u.isRebuildCurMinNumKxQFlag());
+  }
+
   private static double errorFactor(int lgK, boolean oooFlag, double numStdDev) {
     double f;
     if (oooFlag) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 02/10: Interim

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit be8dbfe7d33502b3c56af7cb67e70d0775ac4753
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Dec 19 11:54:32 2019 -0800

    Interim
---
 src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
index e137d63..04fcc0b 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
@@ -36,7 +36,7 @@ import org.apache.datasketches.memory.WritableMemory;
  * @author Lee Rhodes
  */
 class DirectAuxHashMap implements AuxHashMap {
-  private final DirectHllArray host; //hosts the Updatable Memory
+  private final DirectHllArray host; //hosts the WritableMemory and read-only Memory
 
   DirectAuxHashMap(final DirectHllArray host, final boolean initialize) {
     this.host = host;
@@ -47,7 +47,7 @@ class DirectAuxHashMap implements AuxHashMap {
     } else {
       if (extractLgArr(host.mem) < initLgArrInts) {
         final int lgArr =
-            PreambleUtil.computeLgArr(host.wmem, host.auxHashMap.getAuxCount(), host.lgConfigK);
+            PreambleUtil.computeLgArr(host.mem, host.auxHashMap.getAuxCount(), host.lgConfigK);
         insertLgArr(host.wmem, lgArr);
       }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org


[incubator-datasketches-java] 04/10: Remove troubleshooting tests

Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

leerho pushed a commit to branch NewHllIterator
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-java.git

commit a5bf06a9c65fff65f502dc04ab9c8d1fafc83545
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Thu Dec 19 12:34:07 2019 -0800

    Remove troubleshooting tests
---
 src/main/java/org/apache/datasketches/hll/Conversions.java     |  2 +-
 .../java/org/apache/datasketches/hll/DirectAuxHashMap.java     |  9 +++++++--
 src/main/java/org/apache/datasketches/hll/Union.java           |  8 ++++----
 .../java/org/apache/datasketches/hll/ToFromByteArrayTest.java  | 10 ++--------
 4 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/hll/Conversions.java b/src/main/java/org/apache/datasketches/hll/Conversions.java
index 35eaa0a..a2ad780 100644
--- a/src/main/java/org/apache/datasketches/hll/Conversions.java
+++ b/src/main/java/org/apache/datasketches/hll/Conversions.java
@@ -73,7 +73,7 @@ class Conversions {
    * @param absHllArr an instance of AbstractHllArray
    * @return pair values representing numAtCurMin and curMin
    */
-  private static final int curMinAndNum(final AbstractHllArray absHllArr) {
+  static final int curMinAndNum(final AbstractHllArray absHllArr) {
     int curMin = 64;
     int numAtCurMin = 0;
     final PairIterator itr = absHllArr.iterator();
diff --git a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
index 04fcc0b..3eba9f9 100644
--- a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
+++ b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java
@@ -41,13 +41,18 @@ class DirectAuxHashMap implements AuxHashMap {
   DirectAuxHashMap(final DirectHllArray host, final boolean initialize) {
     this.host = host;
     final int initLgArrInts = HllUtil.LG_AUX_ARR_INTS[host.lgConfigK];
-    if (initialize) {
+    if (initialize) { //must be writable
       insertLgArr(host.wmem, initLgArrInts);
       host.wmem.clear(host.auxStart, 4 << initLgArrInts);
     } else {
       if (extractLgArr(host.mem) < initLgArrInts) {
+        if (host.wmem == null) { //read-only
+          throw new SketchesArgumentException(
+              "Possible Memory image corruption, missing LgArr field in preamble.");
+        }
+        //insert the correct LgArr value
         final int lgArr =
-            PreambleUtil.computeLgArr(host.mem, host.auxHashMap.getAuxCount(), host.lgConfigK);
+            PreambleUtil.computeLgArr(host.wmem, host.auxHashMap.getAuxCount(), host.lgConfigK);
         insertLgArr(host.wmem, lgArr);
       }
     }
diff --git a/src/main/java/org/apache/datasketches/hll/Union.java b/src/main/java/org/apache/datasketches/hll/Union.java
index cacde4b..17878e9 100644
--- a/src/main/java/org/apache/datasketches/hll/Union.java
+++ b/src/main/java/org/apache/datasketches/hll/Union.java
@@ -415,8 +415,8 @@ public class Union extends BaseHllSketch {
       case 4:  //src <= max, src >= gdt, gdtHLL, gdtHeap,   forward merge, no downsample, ooof=True
       {
         if ((srcLgK == gadgetLgK) && (source.getTgtHllType() == HLL_8) && (!source.isMemory())) {
-          source.mergeTo(gadget);
-          //specialMerge(source, gadget);
+          //source.mergeTo(gadget);
+          specialMerge(source, gadget);
         } else {
           source.mergeTo(gadget);    //merge src(Hll?,heap/mem,hll) -> gdt(Hll8,heap,hll), autofold
         }
@@ -458,8 +458,8 @@ public class Union extends BaseHllSketch {
       {
         final HllSketch gdtHll8Heap = downsample(gadget, srcLgK); //downsample gdt to srcLgK
         if ((source.getTgtHllType() == HLL_8) && (!source.isMemory())) {
-          source.mergeTo(gdtHll8Heap);
-          //specialMerge(source, gdtHll8Heap);
+          //source.mergeTo(gdtHll8Heap);
+          specialMerge(source, gdtHll8Heap);
         } else {
           source.mergeTo(gdtHll8Heap);//merge src(Hll?,heap/mem,hll) -> gdt(Hll8,heap,hll), autofold
         }
diff --git a/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java b/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
index ad76efb..d2530e5 100644
--- a/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
+++ b/src/test/java/org/apache/datasketches/hll/ToFromByteArrayTest.java
@@ -36,7 +36,6 @@ public class ToFromByteArrayTest {
 
   static final int[] nArr = new int[] {1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000};
 
-
   @Test
   public void checkToFromSketch1() {
     for (int i = 0; i < 10; i++) {
@@ -76,11 +75,6 @@ public class ToFromByteArrayTest {
   }
 
   @Test
-  public void detailCheck() {
-    toFrom2(10, HLL_4, 24322); //24322 fails
-  }
-
-  @Test
   public void checkToFromSketch2() {
     for (int i = 0; i < 10; i++) {
       int n = nArr[i];
@@ -140,10 +134,10 @@ public class ToFromByteArrayTest {
   }
 
   /**
-   * @param s value to print
+   * @param o value to print
    */
   static void println(Object o) {
-    System.out.println(o.toString()); //disable here
+    //System.out.println(o.toString()); //disable here
   }
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org