You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2025/07/03 17:05:40 UTC
(datasketches-java) branch ffm_phase2 updated: Commit finished hll2 changes to FFM
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch ffm_phase2
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
The following commit(s) were added to refs/heads/ffm_phase2 by this push:
new e91894eac Commit finished hll2 changes to FFM
e91894eac is described below
commit e91894eac5480589467a45306f0e359b44de70d6
Author: Lee Rhodes <le...@gmail.com>
AuthorDate: Thu Jul 3 10:05:22 2025 -0700
Commit finished hll2 changes to FFM
---
.../datasketches/common/MemorySegmentStatus.java | 39 +-
.../java/org/apache/datasketches/common/Util.java | 91 +--
.../apache/datasketches/hll2/AbstractCoupons.java | 160 ++++
.../apache/datasketches/hll2/AbstractHllArray.java | 252 +++++++
.../org/apache/datasketches/hll2/AuxHashMap.java | 79 ++
.../apache/datasketches/hll2/BaseHllSketch.java | 416 +++++++++++
.../hll2/CompositeInterpolationXTable.java | 782 +++++++++++++++++++
.../org/apache/datasketches/hll2/Conversions.java | 135 ++++
.../apache/datasketches/hll2/CouponHashSet.java | 165 ++++
.../org/apache/datasketches/hll2/CouponList.java | 237 ++++++
.../apache/datasketches/hll2/CouponMapping.java | 58 ++
.../datasketches/hll2/CubicInterpolation.java | 165 ++++
.../java/org/apache/datasketches/hll2/CurMode.java | 41 +
.../apache/datasketches/hll2/DirectAuxHashMap.java | 218 ++++++
.../datasketches/hll2/DirectCouponHashSet.java | 163 ++++
.../apache/datasketches/hll2/DirectCouponList.java | 321 ++++++++
.../apache/datasketches/hll2/DirectHll4Array.java | 209 ++++++
.../apache/datasketches/hll2/DirectHll6Array.java | 145 ++++
.../apache/datasketches/hll2/DirectHll8Array.java | 128 ++++
.../apache/datasketches/hll2/DirectHllArray.java | 292 ++++++++
.../apache/datasketches/hll2/HarmonicNumbers.java | 94 +++
.../apache/datasketches/hll2/HeapAuxHashMap.java | 231 ++++++
.../org/apache/datasketches/hll2/Hll4Array.java | 180 +++++
.../org/apache/datasketches/hll2/Hll4Update.java | 201 +++++
.../org/apache/datasketches/hll2/Hll6Array.java | 156 ++++
.../org/apache/datasketches/hll2/Hll8Array.java | 143 ++++
.../org/apache/datasketches/hll2/HllArray.java | 263 +++++++
.../apache/datasketches/hll2/HllEstimators.java | 156 ++++
.../apache/datasketches/hll2/HllPairIterator.java | 100 +++
.../org/apache/datasketches/hll2/HllSketch.java | 566 ++++++++++++++
.../apache/datasketches/hll2/HllSketchImpl.java | 120 +++
.../java/org/apache/datasketches/hll2/HllUtil.java | 144 ++++
.../datasketches/hll2/IntArrayPairIterator.java | 91 +++
.../hll2/IntMemorySegmentPairIterator.java | 105 +++
.../org/apache/datasketches/hll2/PairIterator.java | 98 +++
.../org/apache/datasketches/hll2/PreambleUtil.java | 473 ++++++++++++
.../datasketches/hll2/RelativeErrorTables.java | 119 +++
.../org/apache/datasketches/hll2/TgtHllType.java | 77 ++
.../apache/datasketches/hll2/ToByteArrayImpl.java | 237 ++++++
.../java/org/apache/datasketches/hll2/Union.java | 826 +++++++++++++++++++++
.../hll2/doc-files/HLL_HIP_K12T20U20.png | Bin 0 -> 155358 bytes
.../hll2/doc-files/HLL_UnionTime4_6_8_Java_CPP.png | Bin 0 -> 321106 bytes
.../org/apache/datasketches/hll2/package-info.java | 160 ++++
.../org/apache/datasketches/theta/AnotBimpl.java | 20 +-
.../apache/datasketches/theta/CompactSketch.java | 16 +-
.../theta/ConcurrentHeapThetaBuffer.java | 4 +-
.../datasketches/theta/DirectCompactSketch.java | 10 +-
.../theta/DirectQuickSelectSketchR.java | 11 +-
.../datasketches/theta/IntersectionImpl.java | 31 +-
...MemorySegmentCompactCompressedHashIterator.java | 21 +-
.../apache/datasketches/theta/SetOperation.java | 12 +-
.../org/apache/datasketches/theta/UnionImpl.java | 12 +-
.../apache/datasketches/theta/UpdateSketch.java | 12 +-
.../apache/datasketches/hll2/AuxHashMapTest.java | 102 +++
.../datasketches/hll2/BaseHllSketchTest.java | 121 +++
.../apache/datasketches/hll2/CouponListTest.java | 160 ++++
.../datasketches/hll2/CrossCountingTest.java | 136 ++++
.../datasketches/hll2/DirectAuxHashMapTest.java | 347 +++++++++
.../datasketches/hll2/DirectCouponListTest.java | 235 ++++++
.../datasketches/hll2/DirectHllSketchTest.java | 172 +++++
.../apache/datasketches/hll2/DirectUnionTest.java | 457 ++++++++++++
.../org/apache/datasketches/hll2/HllArrayTest.java | 272 +++++++
.../hll2/HllSketchCrossLanguageTest.java | 102 +++
.../apache/datasketches/hll2/HllSketchTest.java | 458 ++++++++++++
.../apache/datasketches/hll2/IsomorphicTest.java | 392 ++++++++++
.../apache/datasketches/hll2/PreambleUtilTest.java | 197 +++++
.../datasketches/hll2/SizeAndModeTransitions.java | 103 +++
.../org/apache/datasketches/hll2/TablesTest.java | 74 ++
.../datasketches/hll2/ToFromByteArrayTest.java | 142 ++++
.../apache/datasketches/hll2/UnionCaseTest.java | 450 +++++++++++
.../org/apache/datasketches/hll2/UnionTest.java | 525 +++++++++++++
.../datasketches/theta/CompactSketchTest.java | 242 +++---
.../ConcurrentDirectQuickSelectSketchTest.java | 418 +++++------
.../theta/ConcurrentHeapQuickSelectSketchTest.java | 360 ++++-----
.../theta/DirectQuickSelectSketchTest.java | 432 +++++------
.../theta/ForwardCompatibilityTest.java | 100 +--
.../theta/HeapQuickSelectSketchTest.java | 286 +++----
.../theta/HeapifyWrapSerVer1and2Test.java | 166 ++---
.../datasketches/theta/SetOpsCornerCasesTest.java | 223 +++---
.../datasketches/theta/SingleItemSketchTest.java | 108 +--
.../org/apache/datasketches/theta/SketchTest.java | 196 ++---
.../apache/datasketches/theta/UnionImplTest.java | 7 +-
82 files changed, 14349 insertions(+), 1419 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java b/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java
index 889e8c0b8..f9fb793f4 100644
--- a/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java
+++ b/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java
@@ -37,10 +37,10 @@ public interface MemorySegmentStatus {
* Returns true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment.
* @return true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment.
*/
- boolean isDirect();
+ boolean isOffHeap();
/**
- * Returns true if the backing MemorySegment of this object refers to the same MemorySegment of <i>that</i>.
+ * Returns true if an internally referenced MemorySegment refers to the same MemorySegment as <i>that</i>.
* They can either have the same off-heap memory location and size, or refer to the same on-heap array object.
*
* <p>If both segment are off-heap, they both must have the same starting address and the same size.</p>
@@ -51,8 +51,41 @@ public interface MemorySegmentStatus {
* <p>Returns false if either argument is null;</p>
*
* @param that The given MemorySegment.
- * @return true if the backing MemorySegment of this object hierarchy refers to the same MemorySegment of <i>that</i>.
+ * @return true if an internally referenced MemorySegment refers to the same MemorySegment as <i>that</i>.
*/
boolean isSameResource(final MemorySegment that);
+ /**
+ * Returns true if the two given MemorySegments refer to the same backing resource,
+ * which is either an off-heap memory address and size, or the same on-heap array object.
+ *
+ * <p>If both segment are off-heap, they both must have the same starting address and the same size.</p>
+ *
+ * <p>For on-heap segments, both segments must be based on or derived from the same array object and neither segment
+ * can be read-only.</p>
+ *
+ * <p>Returns false if either argument is null;</p>
+ *
+ * @param seg1 The first given MemorySegment
+ * @param seg2 The second given MemorySegment
+ * @return true if both MemorySegments are determined to be the same backing memory.
+ */
+ static boolean isSameResource(final MemorySegment seg1, final MemorySegment seg2) {
+ if ((seg1 == null) || (seg2 == null)) { return false; }
+ if (!seg1.scope().isAlive() || !seg2.scope().isAlive()) {
+ throw new IllegalArgumentException("Both arguments must be alive.");
+ }
+ final boolean seg1Native = seg1.isNative();
+ final boolean seg2Native = seg2.isNative();
+ if (seg1Native ^ seg2Native) { return false; }
+ if (seg1Native && seg2Native) { //both off heap
+ return (seg1.address() == seg2.address()) && (seg1.byteSize() == seg2.byteSize());
+ }
+ //both on heap
+ if (seg1.isReadOnly() || seg2.isReadOnly()) {
+ throw new IllegalArgumentException("Cannot determine 'isSameBackingMemory(..)' on heap if either MemorySegment is Read-only.");
+ }
+ return (seg1.heapBase().orElse(null) == seg2.heapBase().orElse(null));
+ }
+
}
diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java
index 5a993591f..bc0270504 100644
--- a/src/main/java/org/apache/datasketches/common/Util.java
+++ b/src/main/java/org/apache/datasketches/common/Util.java
@@ -101,10 +101,10 @@ public final class Util {
* @return an int extracted from a Little-Endian byte array.
*/
public static int bytesToInt(final byte[] arr) {
- return arr[3] << 24
- | (arr[2] & 0xff) << 16
- | (arr[1] & 0xff) << 8
- | arr[0] & 0xff;
+ return (arr[3] << 24)
+ | ((arr[2] & 0xff) << 16)
+ | ((arr[1] & 0xff) << 8)
+ | (arr[0] & 0xff);
}
/**
@@ -113,14 +113,14 @@ public final class Util {
* @return a long extracted from a Little-Endian byte array.
*/
public static long bytesToLong(final byte[] arr) {
- return (long)arr[7] << 56
- | ((long)arr[6] & 0xff) << 48
- | ((long)arr[5] & 0xff) << 40
- | ((long)arr[4] & 0xff) << 32
- | ((long)arr[3] & 0xff) << 24
- | ((long)arr[2] & 0xff) << 16
- | ((long)arr[1] & 0xff) << 8
- | (long)arr[0] & 0xff;
+ return ((long)arr[7] << 56)
+ | (((long)arr[6] & 0xff) << 48)
+ | (((long)arr[5] & 0xff) << 40)
+ | (((long)arr[4] & 0xff) << 32)
+ | (((long)arr[3] & 0xff) << 24)
+ | (((long)arr[2] & 0xff) << 16)
+ | (((long)arr[1] & 0xff) << 8)
+ | ((long)arr[0] & 0xff);
}
/**
@@ -159,7 +159,7 @@ public final class Util {
static long[] convertToLongArray(final byte[] byteArr, final boolean littleEndian) {
final int len = byteArr.length;
- final long[] longArr = new long[len / 8 + (len % 8 != 0 ? 1 : 0)];
+ final long[] longArr = new long[(len / 8) + ((len % 8) != 0 ? 1 : 0)];
int off = 0;
int longArrIdx = 0;
while (off < len) {
@@ -191,7 +191,7 @@ public final class Util {
final long mask = 0XFFL;
final StringBuilder sb = new StringBuilder();
for (int i = 8; i-- > 0; ) {
- final String s = Long.toHexString(v >>> i * 8 & mask);
+ final String s = Long.toHexString((v >>> (i * 8)) & mask);
sb.append(zeroPad(s, 2)).append(" ");
}
return sb.toString();
@@ -211,7 +211,7 @@ public final class Util {
final int mask = signed ? 0XFFFFFFFF : 0XFF;
final int arrLen = arr.length;
if (littleEndian) {
- for (int i = 0; i < arrLen - 1; i++) {
+ for (int i = 0; i < (arrLen - 1); i++) {
sb.append(arr[i] & mask).append(sep);
}
sb.append(arr[arrLen - 1] & mask);
@@ -231,8 +231,8 @@ public final class Util {
*/
public static String nanoSecToString(final long nS) {
final long rem_nS = (long)(nS % 1000.0);
- final long rem_uS = (long)(nS / 1000.0 % 1000.0);
- final long rem_mS = (long)(nS / 1000000.0 % 1000.0);
+ final long rem_uS = (long)((nS / 1000.0) % 1000.0);
+ final long rem_mS = (long)((nS / 1000000.0) % 1000.0);
final long sec = (long)(nS / 1000000000.0);
final String nSstr = zeroPad(Long.toString(rem_nS), 3);
final String uSstr = zeroPad(Long.toString(rem_uS), 3);
@@ -247,8 +247,8 @@ public final class Util {
*/
public static String milliSecToString(final long mS) {
final long rem_mS = (long)(mS % 1000.0);
- final long rem_sec = (long)(mS / 1000.0 % 60.0);
- final long rem_min = (long)(mS / 60000.0 % 60.0);
+ final long rem_sec = (long)((mS / 1000.0) % 60.0);
+ final long rem_min = (long)((mS / 60000.0) % 60.0);
final long hr = (long)(mS / 3600000.0);
final String mSstr = zeroPad(Long.toString(rem_mS), 3);
final String secStr = zeroPad(Long.toString(rem_sec), 2);
@@ -296,7 +296,7 @@ public final class Util {
* @param argName This name will be part of the error message if the check fails.
*/
public static void checkIfMultipleOf8AndGT0(final long v, final String argName) {
- if ((v & 0X7L) == 0L && v > 0L) {
+ if (((v & 0X7L) == 0L) && (v > 0L)) {
return;
}
throw new SketchesArgumentException("The value of the parameter \"" + argName
@@ -309,7 +309,7 @@ public final class Util {
* @return true if v is a multiple of 8 and greater than zero
*/
public static boolean isMultipleOf8AndGT0(final long v) {
- return (v & 0X7L) == 0L && v > 0L;
+ return ((v & 0X7L) == 0L) && (v > 0L);
}
//Powers of 2 or powers of base related
@@ -356,7 +356,7 @@ public final class Util {
public static int ceilingPowerOf2(final int n) {
if (n <= 1) { return 1; }
final int topIntPwrOf2 = 1 << 30;
- return n >= topIntPwrOf2 ? topIntPwrOf2 : Integer.highestOneBit(n - 1 << 1);
+ return n >= topIntPwrOf2 ? topIntPwrOf2 : Integer.highestOneBit((n - 1) << 1);
}
/**
@@ -377,7 +377,7 @@ public final class Util {
public static long ceilingPowerOf2(final long n) {
if (n <= 1L) { return 1L; }
final long topIntPwrOf2 = 1L << 62;
- return n >= topIntPwrOf2 ? topIntPwrOf2 : Long.highestOneBit(n - 1L << 1);
+ return n >= topIntPwrOf2 ? topIntPwrOf2 : Long.highestOneBit((n - 1L) << 1);
}
/**
@@ -430,8 +430,8 @@ public final class Util {
* @return the inverse integer power of 2: 1/(2^e) = 2^(-e)
*/
public static double invPow2(final int e) {
- assert (e | 1024 - e - 1) >= 0 : "e cannot be negative or greater than 1023: " + e;
- return Double.longBitsToDouble(1023L - e << 52);
+ assert (e | (1024 - e - 1)) >= 0 : "e cannot be negative or greater than 1023: " + e;
+ return Double.longBitsToDouble((1023L - e) << 52);
}
/**
@@ -695,7 +695,7 @@ public final class Util {
* @param argName Used in the thrown exception.
*/
public static void checkProbability(final double p, final String argName) {
- if (p >= 0.0 && p <= 1.0) {
+ if ((p >= 0.0) && (p <= 1.0)) {
return;
}
throw new SketchesArgumentException("The value of the parameter \"" + argName
@@ -711,7 +711,7 @@ public final class Util {
* @return true if n1 > n2.
*/
public static boolean isLessThanUnsigned(final long n1, final long n2) {
- return n1 < n2 ^ n1 < 0 != n2 < 0;
+ return (n1 < n2) ^ ((n1 < 0) != (n2 < 0));
}
/**
@@ -741,7 +741,7 @@ public final class Util {
* bit is at position zero.
* @return a one if the bit at bitPos is a one, otherwise zero.
*/
- public static final int bitAt(final long number, final int bitPos) {
+ public static int bitAt(final long number, final int bitPos) {
return (number & (1L << bitPos)) > 0 ? 1 : 0;
}
@@ -751,7 +751,7 @@ public final class Util {
* @return the number of decimal digits of the number n
*/
public static int numDigits(long n) {
- if (n % 10 == 0) { n++; }
+ if ((n % 10) == 0) { n++; }
return (int) ceil(log(n) / log(10));
}
@@ -898,39 +898,6 @@ public final class Util {
slice.fill(value);
}
- /**
- * Returns true if the two given MemorySegments refer to the same backing resource,
- * which is either an off-heap memory location and size, or the same on-heap array object.
- *
- * <p>If both segment are off-heap, they both must have the same starting address and the same size.</p>
- *
- * <p>For on-heap segments, both segments must be based on or derived from the same array object and neither segment
- * can be read-only.</p>
- *
- * <p>Returns false if either argument is null;</p>
- *
- * @param seg1 The first given MemorySegment
- * @param seg2 The second given MemorySegment
- * @return true if both MemorySegments are determined to be the same backing memory.
- */
- public static boolean isSameResource(final MemorySegment seg1, final MemorySegment seg2) {
- if ((seg1 == null) || (seg2 == null)) { return false; }
- if (!seg1.scope().isAlive() || !seg2.scope().isAlive()) {
- throw new IllegalArgumentException("Both arguments must be alive.");
- }
- final boolean seg1Native = seg1.isNative();
- final boolean seg2Native = seg2.isNative();
- if (seg1Native ^ seg2Native) { return false; }
- if (seg1Native && seg2Native) { //both off heap
- return (seg1.address() == seg2.address()) && (seg1.byteSize() == seg2.byteSize());
- }
- //both on heap
- if (seg1.isReadOnly() || seg2.isReadOnly()) {
- throw new IllegalArgumentException("Cannot determine 'isSameBackingMemory(..)' on heap if either MemorySegment is Read-only.");
- }
- return (seg1.heapBase().orElse(null) == seg2.heapBase().orElse(null));
- }
-
/**
* Request a new heap MemorySegment with the given capacityBytes and either 8-byte aligned or one byte aligned.
*
diff --git a/src/main/java/org/apache/datasketches/hll2/AbstractCoupons.java b/src/main/java/org/apache/datasketches/hll2/AbstractCoupons.java
new file mode 100644
index 000000000..5f229cee8
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/AbstractCoupons.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.Math.max;
+import static org.apache.datasketches.hll2.HllUtil.COUPON_RSE;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.KEY_MASK_26;
+import static org.apache.datasketches.hll2.ToByteArrayImpl.toCouponByteArray;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+
+/**
+ * @author Lee Rhodes
+ */
+abstract class AbstractCoupons extends HllSketchImpl {
+
+ AbstractCoupons(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode) {
+ super(lgConfigK, tgtHllType, curMode);
+ }
+
+ @Override
+ double getCompositeEstimate() {
+ return getEstimate();
+ }
+
+ abstract int getCouponCount();
+
+ abstract int[] getCouponIntArr();
+
+ /**
+ * This is the estimator for the Coupon List mode and Coupon Hash Set mode.
+ *
+ * <p>Note: This is an approximation to the true mapping from numCoupons to N,
+ * which has a range of validity roughly from 0 to 6 million coupons.</p>
+ *
+ * <p>The k of the implied coupon sketch, which must not be confused with the k of the HLL
+ * sketch. In this application k is always 2^26, which is the number of address bits of the
+ * 32-bit coupon.</p>
+ * @return the unique count estimate.
+ */
+ @Override
+ double getEstimate() {
+ final int couponCount = getCouponCount();
+ final double est = CubicInterpolation.usingXAndYTables(CouponMapping.xArr,
+ CouponMapping.yArr, couponCount);
+ return max(est, couponCount);
+ }
+
+ @Override
+ double getHipEstimate() {
+ return getEstimate();
+ }
+
+ abstract int getLgCouponArrInts();
+
+ @Override
+ double getLowerBound(final int numStdDev) {
+ HllUtil.checkNumStdDev(numStdDev);
+ final int couponCount = getCouponCount();
+ final double est = CubicInterpolation.usingXAndYTables(CouponMapping.xArr,
+ CouponMapping.yArr, couponCount);
+ final double tmp = est / (1.0 + (numStdDev * COUPON_RSE));
+ return max(tmp, couponCount);
+ }
+
+ @Override
+ double getUpperBound(final int numStdDev) {
+ HllUtil.checkNumStdDev(numStdDev);
+ final int couponCount = getCouponCount();
+ final double est = CubicInterpolation.usingXAndYTables(CouponMapping.xArr,
+ CouponMapping.yArr, couponCount);
+ final double tmp = est / (1.0 - (numStdDev * COUPON_RSE));
+ return max(tmp, couponCount);
+ }
+
+ @Override
+ int getUpdatableSerializationBytes() {
+ return getSegDataStart() + (4 << getLgCouponArrInts());
+ }
+
+ @Override
+ boolean isEmpty() {
+ return getCouponCount() == 0;
+ }
+
+ @Override
+ boolean isOutOfOrder() {
+ return false;
+ }
+
+ @Override
+ boolean isRebuildCurMinNumKxQFlag() {
+ return false;
+ }
+
+ @Override
+ void putEmptyFlag(final boolean empty) {} //no-op for coupons
+
+ @Override
+ void putOutOfOrder(final boolean outOfOrder) {} //no-op for coupons
+
+ @Override
+ void putRebuildCurMinNumKxQFlag(final boolean rebuild) {} //no-op for coupons
+
+ @Override
+ byte[] toCompactByteArray() {
+ return toCouponByteArray(this, true);
+ }
+
+ @Override
+ byte[] toUpdatableByteArray() {
+ return toCouponByteArray(this, false);
+ }
+
+ //FIND for Heap and Direct
+ //Searches the Coupon hash table for an empty slot or a duplicate depending on the context.
+ //If entire entry is empty, returns one's complement of index = found empty.
+ //If entry equals given coupon, returns its index = found duplicate coupon
+ //Continues searching
+ //If the probe comes back to original index, throws an exception.
+ //Called by CouponHashSet.couponUpdate()
+ //Called by CouponHashSet.growHashSet()
+ //Called by DirectCouponHashSet.growHashSet()
+ static final int find(final int[] array, final int lgArrInts, final int coupon) {
+ final int arrMask = array.length - 1;
+ int probe = coupon & arrMask;
+ final int loopIndex = probe;
+ do {
+ final int couponAtIdx = array[probe];
+ if (couponAtIdx == EMPTY) {
+ return ~probe; //empty
+ }
+ else if (coupon == couponAtIdx) {
+ return probe; //duplicate
+ }
+ final int stride = ((coupon & KEY_MASK_26) >>> lgArrInts) | 1;
+ probe = (probe + stride) & arrMask;
+ } while (probe != loopIndex);
+ throw new SketchesArgumentException("Key not found and no empty slots!");
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/AbstractHllArray.java b/src/main/java/org/apache/datasketches/hll2/AbstractHllArray.java
new file mode 100644
index 000000000..11c440e4b
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/AbstractHllArray.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.common.Util.invPow2;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_PREINTS;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_4;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_6;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+abstract class AbstractHllArray extends HllSketchImpl {
+ AuxHashMap auxHashMap = null; //used for both heap and direct HLL4
+ final int auxStart; //used for direct HLL4
+
+ AbstractHllArray(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode) {
+ super(lgConfigK, tgtHllType, curMode);
+ auxStart = HLL_BYTE_ARR_START + hll4ArrBytes(lgConfigK);
+ }
+
+ abstract void addToHipAccum(double delta);
+
+ @Override
+ HllArray copyAs(final TgtHllType tgtHllType) {
+ if (tgtHllType == getTgtHllType()) {
+ return (HllArray) copy();
+ }
+ if (tgtHllType == HLL_4) {
+ return Conversions.convertToHll4(this);
+ }
+ if (tgtHllType == HLL_6) {
+ return Conversions.convertToHll6(this);
+ }
+ return Conversions.convertToHll8(this);
+ }
+
+ abstract void decNumAtCurMin();
+
+ AuxHashMap getAuxHashMap() {
+ return auxHashMap;
+ }
+
+ PairIterator getAuxIterator() {
+ return (auxHashMap == null) ? null : auxHashMap.getIterator();
+ }
+
+ @Override
+ int getCompactSerializationBytes() {
+ final AuxHashMap auxHashMap = getAuxHashMap();
+ final int auxCountBytes = (auxHashMap == null) ? 0 : auxHashMap.getAuxCount() << 2;
+ return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxCountBytes;
+ }
+
+ /**
+ * This is the (non-HIP) estimator.
+ * It is called "composite" because multiple estimators are pasted together.
+ * @return the composite estimate
+ */
+ //In C: again-two-registers.c hhb_get_composite_estimate L1489
+ @Override
+ double getCompositeEstimate() {
+ return HllEstimators.hllCompositeEstimate(this);
+ }
+
+ abstract int getCurMin();
+
+ @Override
+ double getEstimate() {
+ if (isOutOfOrder()) {
+ return getCompositeEstimate();
+ }
+ return getHipAccum();
+ }
+
+ /**
+ * For each actual update of the sketch, where the state of the sketch is changed, this register
+ * tracks the Historical Inverse Probability or HIP. Before the update is recorded this register
+ * is incremented by adding the inverse probability 1/Q (defined below). Since KxQ is scaled by K,
+ * the actual increment is K/KxQ as can be seen in the hipAndKxQIncrementalUpdate(...) method
+ * below.
+ * @return the HIP Accumulator
+ */
+ abstract double getHipAccum();
+
+ @Override
+ double getHipEstimate() {
+ return getHipAccum();
+ }
+
+ abstract int getHllByteArrBytes();
+
+ /**
+ * Q = KxQ/K is the probability that an incoming event can modify the state of the sketch.
+ * KxQ is literally K times Q. The HIP estimator is based on tracking this probability as the
+ * sketch gets populated. It is tracked in the hipAccum register.
+ *
+ * <p>The KxQ registers serve dual purposes: They are used in the HIP estimator and in
+ * the "raw" HLL estimator defined in the Flajolet, et al, 2007 HLL paper. In order to do this,
+ * the way the KxQ registers are computed here differ from how they are defined in the paper.</p>
+ *
+ * <p>The paper Fig 2 defines</p>
+ * <pre>Z := ( sum[j=1,m](2^(-M[j])) )^(-1).</pre>
+ * But the HIP estimator requires a computation of the probability defined above.
+ * We accomplish both by redefining Z as
+ * <pre>Z := ( m + sum[j=1,m](2^(-M[j] - 1)) )^(-1).</pre>
+ * They are mathematically equivalent since:
+ * <pre>m + sum[j=1,m](2^(-M[j] - 1)) == m + sum[j=1,m](2^(-M[j])) - m == sum[j=1,m](2^(-M[j])).</pre>
+ *
+ * @return KxQ0
+ */
+ abstract double getKxQ0();
+
+ /**
+ * This second KxQ register is shifted by 32 bits to give us more than 90 bits of mantissa
+ * precision, which produces more accurate results for very large counts.
+ * @return KxQ1
+ */
+ abstract double getKxQ1();
+
+ @Override
+ double getLowerBound(final int numStdDev) {
+ HllUtil.checkNumStdDev(numStdDev);
+ return HllEstimators.hllLowerBound(this, numStdDev);
+ }
+
+ @Override
+ int getSegDataStart() {
+ return HLL_BYTE_ARR_START;
+ }
+
+ abstract AuxHashMap getNewAuxHashMap();
+
+ /**
+ * Returns the number of slots that have the value CurMin.
+ * If CurMin is 0, then it returns the number of zeros in the array.
+ * @return the number of slots that have the value CurMin.
+ */
+ abstract int getNumAtCurMin();
+
+ @Override
+ int getPreInts() {
+ return HLL_PREINTS;
+ }
+
+ //overridden by Hll4Array and DirectHll4Array
+ abstract int getNibble(int slotNo);
+
+ abstract int getSlotValue(int slotNo);
+
+ @Override //used by HLL6 and HLL8, Overridden by HLL4
+ int getUpdatableSerializationBytes() {
+ return HLL_BYTE_ARR_START + getHllByteArrBytes();
+ }
+
+ @Override
+ double getUpperBound(final int numStdDev) {
+ HllUtil.checkNumStdDev(numStdDev);
+ return HllEstimators.hllUpperBound(this, numStdDev);
+ }
+
+ @Override
+ abstract PairIterator iterator();
+
+ @Override
+ void mergeTo(final HllSketch that) {
+ throw new SketchesStateException("Possible Corruption, improper access.");
+ }
+
+ abstract void putAuxHashMap(AuxHashMap auxHashMap, boolean compact);
+
+ abstract void putCurMin(int curMin);
+
+ abstract void putHipAccum(double hipAccum);
+
+ abstract void putKxQ0(double kxq0);
+
+ abstract void putKxQ1(double kxq1);
+
+ //overridden by Hll4Array and DirectHll4Array
+ abstract void putNibble(int slotNo, int nibValue);
+
+ abstract void putNumAtCurMin(int numAtCurMin);
+
+ abstract void updateSlotWithKxQ(final int slotNo, final int value);
+
+ abstract void updateSlotNoKxQ(final int slotNo, final int value);
+
+ //Compute HLL byte array lengths, used by both heap and direct.
+
+ static final int hll4ArrBytes(final int lgConfigK) {
+ return 1 << (lgConfigK - 1);
+ }
+
+ static final int hll6ArrBytes(final int lgConfigK) {
+ final int numSlots = 1 << lgConfigK;
+ return ((numSlots * 3) >>> 2) + 1;
+ }
+
+ static final int hll8ArrBytes(final int lgConfigK) {
+ return 1 << lgConfigK;
+ }
+
+ /**
+ * Common HIP and KxQ incremental update for all heap and direct Hll.
+ * This is used when incrementally updating an existing array with non-zero values.
+ * @param host the origin implementation
+ * @param oldValue old value
+ * @param newValue new value
+ */
+ //Called here and by Heap and Direct 6 and 8 bit implementations
+ //In C: again-two-registers.c Lines 851 to 871
+ static final void hipAndKxQIncrementalUpdate(final AbstractHllArray host,
+ final int oldValue, final int newValue) {
+ assert newValue > oldValue;
+ final double kxq0 = host.getKxQ0();
+ final double kxq1 = host.getKxQ1();
+ //update hipAccum BEFORE updating kxq0 and kxq1
+ host.addToHipAccum((1 << host.getLgConfigK()) / (kxq0 + kxq1));
+ incrementalUpdateKxQ(host, oldValue, newValue, kxq0, kxq1);
+ }
+
+ //separate KxQ updates
+ static final void incrementalUpdateKxQ(final AbstractHllArray host,
+ final int oldValue, final int newValue, double kxq0, double kxq1) {
+ //update kxq0 and kxq1; subtract first, then add.
+ if (oldValue < 32) { host.putKxQ0(kxq0 -= invPow2(oldValue)); }
+ else { host.putKxQ1(kxq1 -= invPow2(oldValue)); }
+ if (newValue < 32) { host.putKxQ0(kxq0 + invPow2(newValue)); }
+ else { host.putKxQ1(kxq1 + invPow2(newValue)); }
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/AuxHashMap.java b/src/main/java/org/apache/datasketches/hll2/AuxHashMap.java
new file mode 100644
index 000000000..efe611e1f
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/AuxHashMap.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+import org.apache.datasketches.common.MemorySegmentStatus;
+
+/**
+ * @author Lee Rhodes
+ */
+interface AuxHashMap extends MemorySegmentStatus {
+
+ AuxHashMap copy();
+
+ int getAuxCount();
+
+ int[] getAuxIntArr();
+
+ int getCompactSizeBytes();
+
+ PairIterator getIterator();
+
+ int getLgAuxArrInts();
+
+ int getUpdatableSizeBytes();
+
+ @Override
+ boolean hasMemorySegment();
+
+ @Override
+ boolean isOffHeap();
+
+ @Override
+ boolean isSameResource(MemorySegment seg);
+
+ /**
+ * Adds the slotNo and value to the aux array.
+ * @param slotNo the index from the HLL array
+ * @param value the HLL value at the slotNo.
+ * @throws SketchesStateException if this slotNo already exists in the aux array.
+ */
+ void mustAdd(int slotNo, int value);
+
+ /**
+ * Returns value given slotNo. If this fails an exception is thrown.
+ * @param slotNo the index from the HLL array
+ * @return value the HLL value at the slotNo
+ * @throws SketchesStateException if valid slotNo and value is not found.
+ */
+ int mustFindValueFor(int slotNo);
+
+ /**
+ * Replaces the entry at slotNo with the given value.
+ * @param slotNo the index from the HLL array
+ * @param value the HLL value at the slotNo
+ * @throws SketchesStateException if a valid slotNo, value is not found.
+ */
+ void mustReplace(int slotNo, int value);
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/BaseHllSketch.java b/src/main/java/org/apache/datasketches/hll2/BaseHllSketch.java
new file mode 100644
index 000000000..3cc8cf5c6
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/BaseHllSketch.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.datasketches.hash.MurmurHash3.hash;
+import static org.apache.datasketches.hll2.HllUtil.HLL_HIP_RSE_FACTOR;
+import static org.apache.datasketches.hll2.HllUtil.HLL_NON_HIP_RSE_FACTOR;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.KEY_MASK_26;
+
+import java.lang.foreign.MemorySegment;
+import java.nio.ByteBuffer;
+
+import org.apache.datasketches.common.MemorySegmentStatus;
+import org.apache.datasketches.common.Util;
+
+/**
+ * Although this class is package-private, it provides a single place to define and document
+ * the common public API for both HllSketch and Union.
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+abstract class BaseHllSketch implements MemorySegmentStatus {
+
+ abstract void couponUpdate(int coupon);
+
+ /**
+ * Gets the size in bytes of the current sketch when serialized using
+ * <i>toCompactByteArray()</i>.
+ * @return the size in bytes of the current sketch when serialized using
+ * <i>toCompactByteArray()</i>.
+ */
+ public abstract int getCompactSerializationBytes();
+
+ /**
+ * This is less accurate than the <i>getEstimate()</i> method and is automatically used
+ * when the sketch has gone through union operations where the more accurate HIP estimator
+ * cannot be used.
+ * This is made public only for error characterization software that exists in separate
+ * packages and is not intended for normal use.
+ * @return the composite estimate
+ */
+ public abstract double getCompositeEstimate();
+
+ /**
+ * Returns the current mode of the sketch: LIST, SET, HLL
+ * @return the current mode of the sketch: LIST, SET, HLL
+ */
+ abstract CurMode getCurMode();
+
+ /**
+ * Return the cardinality estimate
+ * @return the cardinality estimate
+ */
+ public abstract double getEstimate();
+
+ /**
+ * Gets the {@link TgtHllType}
+ * @return the TgtHllType enum value
+ */
+ public abstract TgtHllType getTgtHllType();
+
+ /**
+ * Gets the <i>lgConfigK</i>.
+ * @return the <i>lgConfigK</i>.
+ */
+ public abstract int getLgConfigK();
+
+ /**
+ * Gets the approximate lower error bound given the specified number of Standard Deviations.
+ *
+ * @param numStdDev This must be an integer between 1 and 3, inclusive.
+ * <a href="{@docRoot}/resources/dictionary.html#numStdDev">See Number of Standard Deviations</a>
+ * @return the lower bound.
+ */
+ public abstract double getLowerBound(int numStdDev);
+
+ /**
+ * Returns the current serialization version.
+ * @return the current serialization version.
+ */
+ public static final int getSerializationVersion() {
+ return PreambleUtil.SER_VER;
+ }
+
+ /**
+ * Returns the current serialization version of the given MemorySegment.
+ * @param seg the given MemorySegment containing a serialized HllSketch image.
+ * @return the current serialization version.
+ */
+ public static final int getSerializationVersion(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, PreambleUtil.SER_VER_BYTE) & 0XFF;
+ }
+
+ /**
+ * Gets the current (approximate) Relative Error (RE) asymptotic values given several
+ * parameters. This is used primarily for testing.
+ * @param upperBound return the RE for the Upper Bound, otherwise for the Lower Bound.
+ * @param oooFlag set true if the sketch is the result of a non qualifying union operation.
+ * @param lgConfigK the configured value for the sketch.
+ * @param numStdDev the given number of Standard Deviations. This must be an integer between
+ * 1 and 3, inclusive.
+ * <a href="{@docRoot}/resources/dictionary.html#numStdDev">Number of Standard Deviations</a>
+ * @return the current (approximate) RelativeError
+ */
+ public static double getRelErr(final boolean upperBound, final boolean oooFlag,
+ final int lgConfigK, final int numStdDev) {
+ HllUtil.checkLgK(lgConfigK);
+ if (lgConfigK > 12) {
+ final double rseFactor = oooFlag ? HLL_NON_HIP_RSE_FACTOR : HLL_HIP_RSE_FACTOR;
+ final int configK = 1 << lgConfigK;
+ return (numStdDev * rseFactor) / Math.sqrt(configK);
+ }
+ return Math.abs(RelativeErrorTables.getRelErr(upperBound, oooFlag, lgConfigK, numStdDev));
+ }
+
+ /**
+ * Gets the size in bytes of the current sketch when serialized using
+ * <i>toUpdatableByteArray()</i>.
+ * @return the size in bytes of the current sketch when serialized using
+ * <i>toUpdatableByteArray()</i>.
+ */
+ public abstract int getUpdatableSerializationBytes();
+
+ /**
+ * Gets the approximate upper error bound given the specified number of Standard Deviations.
+ *
+ * @param numStdDev This must be an integer between 1 and 3, inclusive.
+ * <a href="{@docRoot}/resources/dictionary.html#numStdDev">Number of Standard Deviations</a>
+ * @return the upper bound.
+ */
+ public abstract double getUpperBound(int numStdDev);
+
+ /**
+ * Returns true if empty
+ * @return true if empty
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * Returns true if the backing MemorySegment of this sketch is in compact form.
+ * @return true if the backing MemorySegment of this sketch is in compact form.
+ */
+ public abstract boolean isCompact();
+
+ /**
+ * This HLL family of sketches and operators is always estimating, even for very small values.
+ * @return true
+ */
+ public boolean isEstimationMode() {
+ return true;
+ }
+
+ /**
+ * Returns true if this sketch was created using MemorySegment.
+ * @return true if this sketch was created using MemorySegment.
+ */
+ @Override
+ public abstract boolean hasMemorySegment();
+
+ /**
+ * Returns true if the backing MemorySegment for this sketch is off-heap.
+ * @return true if the backing MemorySegment for this sketch is off-heap.
+ */
+ @Override
+ public abstract boolean isOffHeap();
+
+ /**
+ * Gets the Out-of-order flag.
+ * @return true if the current estimator is the non-HIP estimator, which is slightly less
+ * accurate than the HIP estimator.
+ */
+ abstract boolean isOutOfOrder();
+
+ /**
+ * Returns true if the given MemorySegment refers to the same underlying resource as this sketch.
+ * The capacities must be the same. If <i>this</i> is a region,
+ * the region offset must also be the same.
+ *
+ * <p>This is only relevant for HLL_4 sketches that have been configured for off-heap
+ * using MemorySegment. For on-heap sketches or unions this will return false.
+ *
+ * <p>It is rare, but possible, the the off-heap MemorySegment that has been allocated to an HLL_4
+ * sketch may not be large enough. If this should happen, the sketch makes a request for more
+ * space from the owner of the resource and then moves itself to this new location. This all
+ * happens transparently to the user. This method provides a means for the user to
+ * inquire of the sketch if it has, in fact, moved itself.
+ *
+ * @param seg the given MemorySegment
+ * @return true if the given MemorySegment refers to the same underlying resource as this sketch or
+ * union.
+ */
+ @Override
+ public abstract boolean isSameResource(MemorySegment seg);
+
+ /**
+ * Resets to empty, but does not change the configured values of lgConfigK and tgtHllType.
+ */
+ public abstract void reset();
+
+ /**
+ * Serializes this sketch as a byte array in compact form. The compact form is smaller in size
+ * than the updatable form and read-only. It can be used in union operations as follows:
+ * <pre>{@code
+ * Union union; HllSketch sk, sk2;
+ * int lgK = 12;
+ * sk = new HllSketch(lgK, TgtHllType.HLL_4); //can be 4, 6, or 8
+ * for (int i = 0; i < (2 << lgK); i++) { sk.update(i); }
+ * byte[] arr = HllSketch.toCompactByteArray();
+ * //...
+ * union = Union.heapify(arr); //initializes the union using data from the array.
+ * //OR, if used in an off-heap environment:
+ * union = Union.heapify(MemorySegment.ofArray(arr)); //same as above, except from MemorySegment object.
+ *
+ * //To recover an updatable heap sketch:
+ * sk2 = HllSketch.heapify(arr);
+ * //OR, if used in an off-heap environment:
+ * sk2 = HllSketch.heapify(MemorySegment.ofArray(arr));
+ * }</pre>
+ *
+ * <p>The sketch "wrapping" operation skips actual deserialization thus is quite fast. However,
+ * any attempt to update the derived HllSketch will result in a Read-only exception.</p>
+ *
+ * <p>Note that in some cases, based on the state of the sketch, the compact form is
+ * indistiguishable from the updatable form. In these cases the updatable form is returned
+ * and the compact flag bit will not be set.</p>
+ * @return this sketch as a compact byte array.
+ */
+ public abstract byte[] toCompactByteArray();
+
+ /**
+ * Serializes this sketch as a byte array in an updatable form. The updatable form is larger than
+ * the compact form. The use of this form is primarily in environments that support updating
+ * sketches in off-heap MemorySegment. If the sketch is constructed using HLL_8, sketch updating and
+ * union updating operations can actually occur in MemorySegment, which can be off-heap:
+ * <pre>{@code
+ * Union union; HllSketch sk;
+ * int lgK = 12;
+ * sk = new HllSketch(lgK, TgtHllType.HLL_8) //must be 8
+ * for (int i = 0; i < (2 << lgK); i++) { sk.update(i); }
+ * byte[] arr = sk.toUpdatableByteArray();
+ * MemorySegment wseg = MemorySegment.wrap(arr);
+ * //...
+ * union = Union.writableWrap(wseg); //no deserialization!
+ * }</pre>
+ * @return this sketch as an updatable byte array.
+ */
+ public abstract byte[] toUpdatableByteArray();
+
+ /**
+ * Human readable summary as a string.
+ * @return Human readable summary as a string.
+ */
+ @Override
+ public String toString() {
+ return toString(true, false, false, false);
+ }
+
+ /**
+ * Human readable summary with optional detail. Does not list empty entries.
+ * @param summary if true, output the sketch summary
+ * @param detail if true, output the internal data array
+ * @param auxDetail if true, output the internal Aux array, if it exists.
+ * @return human readable string with optional detail.
+ */
+ public String toString(final boolean summary, final boolean detail, final boolean auxDetail) {
+ return toString(summary, detail, auxDetail, false);
+ }
+
+ /**
+ * Human readable summary with optional detail
+ * @param summary if true, output the sketch summary
+ * @param detail if true, output the internal data array
+ * @param auxDetail if true, output the internal Aux array, if it exists.
+ * @param all if true, outputs all entries including empty ones
+ * @return human readable string with optional detail.
+ */
+ public abstract String toString(boolean summary, boolean detail, boolean auxDetail,
+ boolean all);
+
+ /**
+ * Present the given long as a potential unique item.
+ *
+ * @param datum The given long datum.
+ */
+ public void update(final long datum) {
+ final long[] data = { datum };
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given double (or float) datum as a potential unique item.
+ * The double will be converted to a long using Double.doubleToLongBits(datum),
+ * which normalizes all NaN values to a single NaN representation.
+ * Plus and minus zero will be normalized to plus zero.
+ * The special floating-point values NaN and +/- Infinity are treated as distinct.
+ *
+ * @param datum The given double datum.
+ */
+ public void update(final double datum) {
+ final double d = (datum == 0.0) ? 0.0 : datum; // canonicalize -0.0, 0.0
+ final long[] data = { Double.doubleToLongBits(d) };// canonicalize all NaN & +/- infinity forms
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given String as a potential unique item.
+ * The string is converted to a byte array using UTF8 encoding.
+ * If the string is null or empty no update attempt is made and the method returns.
+ *
+ * <p>Note: About 2X faster performance can be obtained by first converting the String to a
+ * char[] and updating the sketch with that. This bypasses the complexity of the Java UTF_8
+ * encoding. This, of course, will not produce the same internal hash values as updating directly
+ * with a String. So be consistent! Unioning two sketches, one fed with strings and the other
+ * fed with char[] will be meaningless.
+ * </p>
+ *
+ * @param datum The given String.
+ */
+ public void update(final String datum) {
+ if ((datum == null) || datum.isEmpty()) { return; }
+ final byte[] data = datum.getBytes(UTF_8);
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given byte buffer as a potential unique item.
+ * Bytes are read from the current position of the buffer until its limit.
+ * If the byte buffer is null or has no bytes remaining, no update attempt is made and the method returns.
+ *
+ * <p>This method will not modify the position, mark, limit, or byte order of the buffer.</p>
+ *
+ * <p>Little-endian order is preferred, but not required. This method may perform better if the provided byte
+ * buffer is in little-endian order.</p>
+ *
+ * @param data The given byte buffer.
+ */
+ public void update(final ByteBuffer data) {
+ if ((data == null) || (data.remaining() == 0)) { return; }
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given byte array as a potential unique item.
+ * If the byte array is null or empty no update attempt is made and the method returns.
+ *
+ * @param data The given byte array.
+ */
+ public void update(final byte[] data) {
+ if ((data == null) || (data.length == 0)) { return; }
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given char array as a potential unique item.
+ * If the char array is null or empty no update attempt is made and the method returns.
+ *
+ * <p>Note: this will not produce the same output hash values as the <i>update(String)</i>
+ * method but will be a little faster as it avoids the complexity of the UTF8 encoding.</p>
+ *
+ * @param data The given char array.
+ */
+ public void update(final char[] data) {
+ if ((data == null) || (data.length == 0)) { return; }
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given integer array as a potential unique item.
+ * If the integer array is null or empty no update attempt is made and the method returns.
+ *
+ * @param data The given int array.
+ */
+ public void update(final int[] data) {
+ if ((data == null) || (data.length == 0)) { return; }
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ /**
+ * Present the given long array as a potential unique item.
+ * If the long array is null or empty no update attempt is made and the method returns.
+ *
+ * @param data The given long array.
+ */
+ public void update(final long[] data) {
+ if ((data == null) || (data.length == 0)) { return; }
+ couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED)));
+ }
+
+ private static final int coupon(final long[] hash) {
+ final int addr26 = (int) ((hash[0] & KEY_MASK_26));
+ final int lz = Long.numberOfLeadingZeros(hash[1]);
+ final int value = ((lz > 62 ? 62 : lz) + 1);
+ return (value << KEY_BITS_26) | addr26;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CompositeInterpolationXTable.java b/src/main/java/org/apache/datasketches/hll2/CompositeInterpolationXTable.java
new file mode 100644
index 000000000..f96cd6b87
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CompositeInterpolationXTable.java
@@ -0,0 +1,782 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class CompositeInterpolationXTable {
+
+ //CHECKSTYLE.OFF: LineLength
+ /**
+ * 18 Values, index 0 is LgK = 4, index 17 is LgK = 21.
+ */
+ static final int[] yStrides =
+ {1, 2, 3, 5, 10, 20, 40, 80, 160, 320, 640, 1280, 2560, 5120, 10240, 20480, 40960, 81920 };
+
+ static final double xArrs[][] = {
+ // log K = 4
+ {
+ 10.767999803534, 11.237701481774, 11.722738717438, 12.223246391222,
+ 12.739366773787, 13.271184824495, 13.818759686650, 14.382159835785,
+ 14.961390904922, 15.556414447178, 16.167227058768, 16.793705840034,
+ 17.435831011559, 18.093368824077, 18.766214022468, 19.454114555153,
+ 20.156877376380, 20.874309118151, 21.606085061388, 22.351926834624,
+ 23.111406651437, 23.884222842419, 24.669953053285, 25.468324695415,
+ 26.278740385032, 27.100807701976, 27.934264623663, 28.778498012717,
+ 29.633064579479, 30.497559091830, 31.371415928956, 32.254248998304,
+ 33.145649456017, 34.045084140394, 34.952069032314, 35.866210502243,
+ 36.787128772113, 37.714316067007, 38.647432342061, 39.586020708848,
+ 40.529738186256, 41.478377527525, 42.431435504179, 43.388619067416,
+ 44.349650659292, 45.314111448463, 46.281805382722, 47.252472240229,
+ 48.225969481651, 49.202042512440, 50.180350422249, 51.160761480664,
+ 52.143032445587, 53.127233235412, 54.112986437616, 55.100182392694,
+ 56.088558030701, 57.078156023607, 58.068788167275, 59.060493030644,
+ 60.053033784417, 61.046284072738, 62.040286486621, 63.034929032992,
+ 64.030027371399, 65.025588917110, 66.021614945599, 67.018177418440,
+ 68.015186786528, 69.012467972526, 70.010014322660, 71.007638150468,
+ 72.005472777573, 73.003558069130, 74.001925061314, 75.000323809106,
+ 75.998875046818, 76.997735309722, 77.996356820987, 78.995236661037,
+ 79.994185446564, 80.993179953574, 81.991936172868, 82.991277726952,
+ 83.990564675011, 84.990016215330, 85.989354175963, 86.988664031183,
+ 87.988038383374, 88.987487931153, 89.987011275632, 90.986376635228,
+ 91.985802667485, 92.985125933965, 93.984894049744, 94.984597185523,
+ 95.984374633793, 96.983987641198, 97.983728001025, 98.983151114601,
+ 99.982917278162, 100.982752268263, 101.982610588914, 102.982425129077,
+ 103.982425883214, 104.982193814993, 105.982188025268, 106.982005912713,
+ 107.981949640254, 108.981777381902, 109.981767708570, 110.981785341504,
+ 111.981686149156, 112.981635012054, 113.981557303839, 114.981412817204,
+ 115.981636136290, 116.981451847734, 117.981619278961, 118.981656205677,
+ 119.981462628717, 120.981648197593, 121.981672839888, 122.981536601882,
+ 123.981424780957, 124.981245233627, 125.981297203177, 126.981469882453,
+ 127.981420558510, 128.981470958251, 129.981803124940, 130.981454077005,
+ 131.981307772677, 132.981136552712, 133.981060440468, 134.980764897921,
+ 135.980250123519, 136.979947889981, 137.979778713731, 138.979651075854,
+ 139.979521006237, 140.979619490143, 141.979418214464, 142.979378742690,
+ 143.979006714929, 144.978789085341, 145.978940618746, 146.978699584620,
+ 147.978509489139, 148.978375065557, 149.978382007245, 150.978320879037,
+ 151.978431501891, 152.978357310224, 153.978079953131, 154.977867908315,
+ 155.977897282267, 156.977732141540, 157.977285841428, 158.977381755450,
+ 159.977107808858, 160.976898092463, 161.976945065052, 162.976415653211,
+ 163.976063254486, 164.976168718176, 165.976009634854, 166.975746910144,
+ 167.975365498336, 168.975302031315, 169.975145815395, 170.975048401084,
+ 171.974904933990, 172.974962856090, 173.974307743427, 174.974505556696,
+ 175.974404122880, 176.974154794798, 177.973936891006, 178.973909768514,
+ 179.973629325330, 180.973396428571, 181.973041798012, 182.972922327014,
+ 183.972453839631, 184.972389341638, 185.971978290400, 186.971721489488,
+ 187.971610992921, 188.971432541974, 189.970886522529, 190.970636748690,
+ 191.970543849249, 192.970222044828, 193.969945219163, 194.969816174441,
+ 195.969660267113, 196.969306217626, 197.969283141670, 198.968952697752,
+ 199.968650726389, 200.968771498143, 201.969063388798, 202.968624157712,
+ 203.968385122471, 204.968459873029, 205.968253426841, 206.968128284823,
+ 207.968002441937, 208.967981887571, 209.967853843352, 210.967653026858,
+ 211.967601511385, 212.967479077297, 213.967293002672, 214.967265475369,
+ 215.967098197208, 216.966915139758, 217.966680984700, 218.966156974161,
+ 219.965788928685, 220.965578199561, 221.965550145047, 222.965055562297,
+ 223.965041228123, 224.965088576052, 225.964951649635, 226.964837079061,
+ 227.964949815142, 228.964753278386, 229.964358666956, 230.964158288467,
+ 231.964118659643, 232.964105599719, 233.964013041408, 234.963810178150,
+ 235.963847138572, 236.963634735645, 237.963596910032, 238.963659218725,
+ 239.963498440155, 240.963445597658, 241.963141291971, 242.962684055746,
+ 243.962664757583, 244.962762174588, 245.962862664373, 246.962595237647,
+ 247.962493432533, 248.962292518484, 249.962622934132, 250.962187495200,
+ 251.962038306920, 252.961926735684, 253.961867248643, 254.961699581530,
+ 255.961371028199
+ },
+ // log K = 5
+ {
+ 22.304000309384, 23.261989079230, 24.249656493833, 25.267226901899,
+ 26.314781734430, 27.392414068382, 28.500177845081, 29.638044883897,
+ 30.805888087516, 32.003714838418, 33.231360682863, 34.488477823483,
+ 35.774924122967, 37.090293268866, 38.434292452116, 39.806594197334,
+ 41.206691028159, 42.634002763091, 44.088065403269, 45.568286128520,
+ 47.074066947685, 48.604745105475, 50.159729872619, 51.738339282229,
+ 53.339805618020, 54.963435280656, 56.608464119077, 58.274038434156,
+ 59.959403900602, 61.663926660982, 63.386737477574, 65.127093833107,
+ 66.884091606686, 68.656959698762, 70.445052178435, 72.247581860591,
+ 74.063607663774, 75.892531705810, 77.733879994604, 79.586788509372,
+ 81.450593478752, 83.324645303244, 85.208482496209, 87.101116685537,
+ 89.002152917959, 90.911090999940, 92.827275073114, 94.750224301260,
+ 96.679773715696, 98.615214511699, 100.556211548644, 102.502148428334,
+ 104.452899214344, 106.407767504404, 108.366625832228, 110.329564692634,
+ 112.295481742874, 114.264715056639, 116.236769512562, 118.211214593179,
+ 120.188376370012, 122.167746134232, 124.148833002551, 126.131612799989,
+ 128.116072530208, 130.102155735854, 132.089449244974, 134.077816832908,
+ 136.067404409248, 138.057703527432, 140.049350564796, 142.041756984355,
+ 144.034780624078, 146.028584459531, 148.022894428003, 150.017952482368,
+ 152.013358245376, 154.009157360372, 156.005076955043, 158.001501688538,
+ 159.998043891849, 161.995141096448, 163.992507770330, 165.990019741406,
+ 167.987379363144, 169.985323168231, 171.983371903769, 173.981134136378,
+ 175.979306888599, 177.977458741010, 179.975994045101, 181.974478508488,
+ 183.973329438036, 185.972061903815, 187.971014174496, 189.969582278901,
+ 191.968450510061, 193.967607022497, 195.966726931677, 197.965915341472,
+ 199.964910588293, 201.964427852336, 203.964308576556, 205.963778587196,
+ 207.963244179225, 209.962423156859, 211.961859295409, 213.961171237132,
+ 215.960624048009, 217.959980906604, 219.959156277574, 221.958692298143,
+ 223.958335010302, 225.958329061602, 227.957722024027, 229.957134954397,
+ 231.956897929205, 233.956901572571, 235.956650770311, 237.956593290745,
+ 239.956700470895, 241.956192761863, 243.956003327600, 245.955794901574,
+ 247.955501305446, 249.954960233132, 251.954510187803, 253.954252376340,
+ 255.954011943050, 257.953388008439, 259.953139524693, 261.952918466928,
+ 263.952432910213, 265.952485387126, 267.952763908808, 269.952510701682,
+ 271.952383356830, 273.951921008979, 275.951766755515, 277.951697578478,
+ 279.951700548755, 281.951805228218, 283.951934023417, 285.952140722678,
+ 287.952063941231, 289.951717207670, 291.951294241955, 293.950937111819,
+ 295.950900506385, 297.950320403388, 299.949546547128, 301.949474819234,
+ 303.949155377395, 305.948725010033, 307.948408553357, 309.948120512731,
+ 311.947897212148, 313.947738181662, 315.947255247579, 317.946631669691,
+ 319.946302193308, 321.945855881155, 323.945603079048, 325.944930145876,
+ 327.945048538923, 329.944795565500, 331.944227346318, 333.943705422222,
+ 335.943231918970, 337.942333069602, 339.942354997227, 341.941890952026,
+ 343.941346276351, 345.940653575739, 347.940019001109, 349.940026817951,
+ 351.938944534275, 353.938172407151, 355.937584534690, 357.937041224148,
+ 359.936673631279, 361.936179855216, 363.936105840596, 365.935367434940,
+ 367.935295315770, 369.934793960370, 371.934314961518, 373.933949277377,
+ 375.933420587284, 377.933148188654, 379.932379363314, 381.931900192237,
+ 383.930951130857, 385.930667541762, 387.930025971711, 389.929529183966,
+ 391.929116798157, 393.928361940635, 395.927673611921, 397.927111110912,
+ 399.926483516685, 401.925799510661, 403.924918492428, 405.924540208915,
+ 407.924014467935, 409.923779779206, 411.922979548363, 413.922515268061,
+ 415.922148908645, 417.921643066153, 419.921126587639, 421.920766647684,
+ 423.920642824255, 425.920411492484, 427.920603679205, 429.920445437023,
+ 431.920286509746, 433.919789058392, 435.919408587813, 437.918873200300,
+ 439.918417416574, 441.918192491069, 443.918376503567, 445.917692006023,
+ 447.917132642691, 449.917217997363, 451.916924115487, 453.916662151474,
+ 455.916485288107, 457.915879851633, 459.915842501592, 461.915547354915,
+ 463.914995141359, 465.914585759187, 467.914144928428, 469.913956069337,
+ 471.913046082896, 473.912468624617, 475.912322983182, 477.912118100240,
+ 479.911923664773, 481.911518944025, 483.911023761757, 485.910386539830,
+ 487.910092345284, 489.910153786770, 491.909575245360, 493.908973823502,
+ 495.908577061168, 497.908550221055, 499.908127020659, 501.907710605296,
+ 503.907806972566, 505.907336571662, 507.907081330259, 509.906648846180,
+ 511.906201574892
+ },
+ // log K = 6
+ {
+ 45.375999854524, 46.820948326168, 48.298703712049, 49.809402685314,
+ 51.353165756275, 52.929933710602, 54.539893242468, 56.183033892089,
+ 57.859217068495, 59.568519998303, 61.310801009963, 63.085886926328,
+ 64.893798324934, 66.734230191052, 68.607173306372, 70.512210309984,
+ 72.449265953981, 74.418007054647, 76.417910710559, 78.448929349152,
+ 80.510585932256, 82.602514696500, 84.724061528721, 86.875166381330,
+ 89.055217833675, 91.263852803721, 93.500342620773, 95.764451518395,
+ 98.055566319691, 100.373234086043, 102.716794393386, 105.085599217566,
+ 107.479363461004, 109.897282858060, 112.338907676111, 114.803401103124,
+ 117.290259931022, 119.799265623507, 122.329323908482, 124.879681624600,
+ 127.450299022993, 130.040239267142, 132.648756733593, 135.275366525219,
+ 137.919720233989, 140.580778017562, 143.258128898839, 145.951552244700,
+ 148.660145472460, 151.383257444852, 154.120434694643, 156.871202914301,
+ 159.634634288797, 162.410377522877, 165.198249005575, 167.997420145534,
+ 170.807579918337, 173.628020587905, 176.458633641294, 179.298282098581,
+ 182.147449731726, 185.005094444186, 187.870899328937, 190.744709255561,
+ 193.625572419776, 196.513740460772, 199.408599560293, 202.309940032246,
+ 205.217132295894, 208.129828601313, 211.047995735211, 213.970919426104,
+ 216.899109829571, 219.832024990852, 222.769136305806, 225.710258431247,
+ 228.655217514772, 231.603984417526, 234.555642012353, 237.510635882749,
+ 240.468820458907, 243.429903849174, 246.393192036307, 249.359251612075,
+ 252.327416912482, 255.297598046767, 258.269777722144, 261.244337060733,
+ 264.219958654591, 267.197643910088, 270.177202282129, 273.157339433786,
+ 276.139287129066, 279.122276848731, 282.107226383752, 285.092136000673,
+ 288.078230550129, 291.065256815999, 294.053160971001, 297.042401030971,
+ 300.031784889467, 303.022502412860, 306.013581138287, 309.005124942334,
+ 311.996910583050, 314.989635544162, 317.982189300799, 320.975762085573,
+ 323.969719809182, 326.963672522149, 329.958419674913, 332.953046205989,
+ 335.948338966110, 338.943201657934, 341.938798787180, 344.935321901459,
+ 347.931089897905, 350.927355153871, 353.923731760523, 356.920383399325,
+ 359.916863372939, 362.914245241722, 365.911216314347, 368.908636150615,
+ 371.905028432797, 374.901981729273, 377.899718967915, 380.897391911925,
+ 383.894945428721, 386.892974193389, 389.890517424427, 392.889050804732,
+ 395.886893560179, 398.884852345719, 401.883235933616, 404.881805247652,
+ 407.880049402262, 410.878410789472, 413.876755787854, 416.875140105536,
+ 419.873758096660, 422.872739431833, 425.871627898739, 428.870663138631,
+ 431.869475472357, 434.868378059774, 437.867537661532, 440.866274620049,
+ 443.865086570094, 446.864282236320, 449.863630871851, 452.862772062678,
+ 455.861877807413, 458.860680461949, 461.859728874700, 464.858478829514,
+ 467.857314559298, 470.856707830759, 473.855641079729, 476.854880576501,
+ 479.853812351976, 482.852666825246, 485.851471230749, 488.850997731659,
+ 491.850420535006, 494.849118717584, 497.848645341646, 500.848230362352,
+ 503.847546143645, 506.847285358235, 509.846275998715, 512.845599444922,
+ 515.845441684852, 518.844522015985, 521.843923976006, 524.843281639782,
+ 527.842399994830, 530.841152548296, 533.840788827465, 536.840058715879,
+ 539.839578267778, 542.839076801367, 545.838441853502, 548.837925005355,
+ 551.837456112430, 554.837041819817, 557.836916936797, 560.836116071833,
+ 563.834832988710, 566.834107840572, 569.832763471368, 572.831700689757,
+ 575.832050680783, 578.831190216307, 581.831249402778, 584.830325442080,
+ 587.828993643112, 590.827962761832, 593.827454067745, 596.825833598259,
+ 599.824193205007, 602.822806340644, 605.822276069352, 608.821423024191,
+ 611.820006110523, 614.819016304854, 617.818220240160, 620.817311827644,
+ 623.816286470744, 626.815211383752, 629.815077535810, 632.813925717940,
+ 635.813392704099, 638.812424935595, 641.811289306882, 644.809932622788,
+ 647.808609610738, 650.807771758015, 653.806851303878, 656.805452367508,
+ 659.804133900742, 662.803103749985, 665.801910649522, 668.800969589926,
+ 671.798796938405, 674.798110719805, 677.797843657187, 680.797711918663,
+ 683.796449689643, 686.794998006462, 689.794205385109, 692.792836542916,
+ 695.792195299304, 698.790778974184, 701.790704644168, 704.790168225574,
+ 707.789261680438, 710.788674453657, 713.787207421297, 716.786043303658,
+ 719.784740017888, 722.783523691482, 725.782739693893, 728.781695938105,
+ 731.780295310459, 734.778719103047, 737.778056368816, 740.777014963309,
+ 743.776470087777, 746.774710218241, 749.773897180146, 752.772555359445,
+ 755.771799417720, 758.770328749569, 761.768940347768, 764.768630836408,
+ 767.767399121425
+ },
+ // log K = 7
+ {
+ 91.554623, 93.970317, 96.431113, 98.937264, 101.488909, 104.086010, 106.728780, 109.417062,
+ 112.150914, 114.930279, 117.755065, 120.625143, 123.540431, 126.500927, 129.506371, 132.556551,
+ 135.651450, 138.790477, 141.973357, 145.200176, 148.470504, 151.783478, 155.139376, 158.537455,
+ 161.977357, 165.458477, 168.980630, 172.543258, 176.145951, 179.788203, 183.469266, 187.188166,
+ 190.945533, 194.739618, 198.570460, 202.437447, 206.339379, 210.276276, 214.247092, 218.251243,
+ 222.288169, 226.357728, 230.458810, 234.590263, 238.751826, 242.942859, 247.162518, 251.410098,
+ 255.685517, 259.987202, 264.314569, 268.667458, 273.044873, 277.445418, 281.869296, 286.316796,
+ 290.785471, 295.276055, 299.788143, 304.319743, 308.869691, 313.438842, 318.026761, 322.632499,
+ 327.254588, 331.892954, 336.547483, 341.217702, 345.901880, 350.601214, 355.313731, 360.038868,
+ 364.777348, 369.526476, 374.288795, 379.061811, 383.846122, 388.640850, 393.446218, 398.260691,
+ 403.084626, 407.916053, 412.755659, 417.604951, 422.461448, 427.324128, 432.195028, 437.072269,
+ 441.956190, 446.846167, 451.741463, 456.641868, 461.549478, 466.461280, 471.377098, 476.297446,
+ 481.222717, 486.151860, 491.085033, 496.022133, 500.962856, 505.906586, 510.853868, 515.802981,
+ 520.755460, 525.711123, 530.668833, 535.629910, 540.592334, 545.557764, 550.525455, 555.494082,
+ 560.463996, 565.436164, 570.410847, 575.385933, 580.361996, 585.339352, 590.318151, 595.300216,
+ 600.281543, 605.264482, 610.250114, 615.236608, 620.222025, 625.208433, 630.195367, 635.183997,
+ 640.172271, 645.162589, 650.153647, 655.144276, 660.133923, 665.123818, 670.115789, 675.109169,
+ 680.103270, 685.097349, 690.091960, 695.086028, 700.080326, 705.075670, 710.069697, 715.065265,
+ 720.062426, 725.058529, 730.054916, 735.050560, 740.048508, 745.044790, 750.041067, 755.037833,
+ 760.035090, 765.033371, 770.029865, 775.025946, 780.022408, 785.021534, 790.018699, 795.016742,
+ 800.013947, 805.012680, 810.009483, 815.007921, 820.008589, 825.007658, 830.007411, 835.006892,
+ 840.006917, 845.006362, 850.004606, 855.003367, 860.001219, 864.999491, 869.999131, 874.998190,
+ 879.998460, 884.997482, 889.995952, 894.995130, 899.994555, 904.993474, 909.991434, 914.990519,
+ 919.991564, 924.992234, 929.993901, 934.992310, 939.992013, 944.993341, 949.993576, 954.992981,
+ 959.993548, 964.993781, 969.993129, 974.994050, 979.994706, 984.994372, 989.994395, 994.995339,
+ 999.995544, 1004.995498, 1009.996864, 1014.997829, 1019.996531, 1024.996724, 1029.997574, 1034.998591,
+ 1039.997807, 1044.998954, 1050.000113, 1055.000633, 1060.000845, 1065.000760, 1070.000907, 1075.001248,
+ 1080.001187, 1084.999775, 1089.999105, 1094.999647, 1100.001983, 1105.003468, 1110.004542, 1115.005170,
+ 1120.006517, 1125.006493, 1130.006837, 1135.008369, 1140.006966, 1145.009225, 1150.008497, 1155.009589,
+ 1160.009558, 1165.011395, 1170.009739, 1175.011131, 1180.013303, 1185.012525, 1190.014190, 1195.016399,
+ 1200.016087, 1205.016715, 1210.018008, 1215.018108, 1220.019677, 1225.020269, 1230.019468, 1235.020515,
+ 1240.021850, 1245.020961, 1250.019785, 1255.020253, 1260.020623, 1265.020156, 1270.020625, 1275.019028,
+ 1280.019972
+ },
+ // log K = 8
+ {
+ 183.877784, 188.720097, 193.652407, 198.675131, 203.788397, 208.992238, 214.286620, 219.671580,
+ 225.147277, 230.713559, 236.370208, 242.117078, 247.953569, 253.880078, 259.895869, 266.000405,
+ 272.193694, 278.474457, 284.843102, 291.298228, 297.839692, 304.466434, 311.177471, 317.972674,
+ 324.850719, 331.811076, 338.852902, 345.974709, 353.175880, 360.455237, 367.811613, 375.244484,
+ 382.752472, 390.334218, 397.988777, 405.714596, 413.510671, 421.376147, 429.308973, 437.308197,
+ 445.372302, 453.500121, 461.690796, 469.942249, 478.253652, 486.623402, 495.049662, 503.532193,
+ 512.068670, 520.658371, 529.299186, 537.990886, 546.732097, 555.520093, 564.355935, 573.235689,
+ 582.159345, 591.126063, 600.133321, 609.181348, 618.267848, 627.392384, 636.552822, 645.748152,
+ 654.977601, 664.239902, 673.534497, 682.859448, 692.213983, 701.598606, 711.009376, 720.446781,
+ 729.910995, 739.397888, 748.908546, 758.443812, 768.000572, 777.577584, 787.174815, 796.791250,
+ 806.427107, 816.079660, 825.748493, 835.435992, 845.136828, 854.854141, 864.583886, 874.328396,
+ 884.085720, 893.855568, 903.637568, 913.431080, 923.234553, 933.049830, 942.873895, 952.707793,
+ 962.551743, 972.402421, 982.263001, 992.131464, 1002.006941, 1011.888460, 1021.777462, 1031.671376,
+ 1041.569600, 1051.476632, 1061.386996, 1071.302834, 1081.224251, 1091.148847, 1101.080297, 1111.013589,
+ 1120.951962, 1130.893360, 1140.838011, 1150.786112, 1160.737100, 1170.689817, 1180.646908, 1190.604979,
+ 1200.565150, 1210.527894, 1220.492169, 1230.460128, 1240.429764, 1250.401749, 1260.373274, 1270.349608,
+ 1280.325544, 1290.302087, 1300.279074, 1310.257850, 1320.238948, 1330.221273, 1340.203554, 1350.186669,
+ 1360.171345, 1370.156233, 1380.142561, 1390.129535, 1400.119177, 1410.107938, 1420.097171, 1430.087586,
+ 1440.077831, 1450.067996, 1460.060299, 1470.050059, 1480.042995, 1490.035101, 1500.026878, 1510.021128,
+ 1520.014262, 1530.008487, 1540.002895, 1549.996342, 1559.991075, 1569.986665, 1579.981202, 1589.976021,
+ 1599.969512, 1609.964051, 1619.959210, 1629.957092, 1639.953516, 1649.948113, 1659.946715, 1669.945253,
+ 1679.941736, 1689.938651, 1699.937600, 1709.936396, 1719.933608, 1729.931040, 1739.930316, 1749.927141,
+ 1759.922600, 1769.920847, 1779.918614, 1789.915707, 1799.913000, 1809.910958, 1819.908532, 1829.906905,
+ 1839.903410, 1849.904217, 1859.901705, 1869.898815, 1879.898869, 1889.899032, 1899.899113, 1909.896164,
+ 1919.893199, 1929.894515, 1939.894965, 1949.893184, 1959.892164, 1969.890794, 1979.889035, 1989.888005,
+ 1999.888538, 2009.888252, 2019.888976, 2029.890242, 2039.890655, 2049.891065, 2059.891068, 2069.889204,
+ 2079.891120, 2089.892462, 2099.892479, 2109.891814, 2119.893049, 2129.893705, 2139.893471, 2149.894162,
+ 2159.894213, 2169.895041, 2179.895987, 2189.894721, 2199.894028, 2209.896005, 2219.898683, 2229.898673,
+ 2239.898012, 2249.898018, 2259.898215, 2269.899380, 2279.902590, 2289.903144, 2299.902047, 2309.904214,
+ 2319.903812, 2329.904180, 2339.902528, 2349.903057, 2359.903089, 2369.904512, 2379.902563, 2389.905522,
+ 2399.903653, 2409.902313, 2419.904452, 2429.905949, 2439.905059, 2449.904954, 2459.904977, 2469.905111,
+ 2479.904049, 2489.903762, 2499.902022, 2509.904352, 2519.903272, 2529.901643, 2539.900789, 2549.903045,
+ 2559.904334
+ },
+ // log K = 9
+ {
+ 368.528954, 378.224619, 388.100156, 398.155846, 408.391912, 418.808631, 429.406377, 440.185048,
+ 451.144613, 462.284693, 473.604932, 485.105306, 496.784844, 508.643147, 520.679581, 532.893098,
+ 545.282616, 557.848295, 570.587645, 583.500268, 596.584345, 609.838697, 623.261582, 636.851681,
+ 650.607666, 664.527320, 678.608336, 692.849674, 707.248088, 721.803089, 736.511331, 751.372034,
+ 766.381798, 781.538178, 796.839808, 812.283881, 827.867995, 843.588520, 859.444318, 875.432109,
+ 891.550599, 907.795630, 924.165943, 940.657393, 957.268601, 973.995685, 990.837235, 1007.789667,
+ 1024.850025, 1042.016602, 1059.286426, 1076.658199, 1094.126523, 1111.689346, 1129.345589, 1147.092119,
+ 1164.926795, 1182.844471, 1200.846880, 1218.929006, 1237.088281, 1255.322930, 1273.630393, 1292.007269,
+ 1310.452948, 1328.963430, 1347.538768, 1366.176566, 1384.871299, 1403.626249, 1422.434094, 1441.295238,
+ 1460.208340, 1479.169853, 1498.179265, 1517.234855, 1536.335403, 1555.477305, 1574.659180, 1593.881592,
+ 1613.139378, 1632.433516, 1651.764587, 1671.127255, 1690.519577, 1709.942250, 1729.393153, 1748.871689,
+ 1768.377142, 1787.907209, 1807.464755, 1827.043146, 1846.641976, 1866.263022, 1885.903553, 1905.564908,
+ 1925.240581, 1944.936723, 1964.648221, 1984.375489, 2004.117246, 2023.875053, 2043.644075, 2063.423514,
+ 2083.218689, 2103.026148, 2122.840475, 2142.669046, 2162.505619, 2182.351758, 2202.205963, 2222.068658,
+ 2241.939010, 2261.814547, 2281.701803, 2301.591894, 2321.490728, 2341.393208, 2361.298412, 2381.211528,
+ 2401.132345, 2421.057721, 2440.985510, 2460.916691, 2480.851468, 2500.792645, 2520.738446, 2540.683395,
+ 2560.634603, 2580.586307, 2600.540530, 2620.497170, 2640.458137, 2660.421783, 2680.386186, 2700.352664,
+ 2720.321411, 2740.291944, 2760.264068, 2780.238910, 2800.212546, 2820.185829, 2840.163952, 2860.142145,
+ 2880.120568, 2900.102466, 2920.081981, 2940.062964, 2960.046262, 2980.029101, 3000.012591, 3019.998424,
+ 3039.983637, 3059.969573, 3079.955927, 3099.942114, 3119.930348, 3139.919614, 3159.909130, 3179.899816,
+ 3199.892231, 3219.885316, 3239.878917, 3259.871675, 3279.865195, 3299.856210, 3319.848871, 3339.840385,
+ 3359.834158, 3379.830660, 3399.826361, 3419.822855, 3439.814714, 3459.813765, 3479.808673, 3499.804339,
+ 3519.797785, 3539.791012, 3559.789885, 3579.785152, 3599.780237, 3619.775480, 3639.772724, 3659.768920,
+ 3679.765735, 3699.766361, 3719.765048, 3739.762845, 3759.761526, 3779.757598, 3799.755719, 3819.751149,
+ 3839.750182, 3859.749673, 3879.749344, 3899.749939, 3919.749060, 3939.748385, 3959.750520, 3979.750055,
+ 3999.750104, 4019.749963, 4039.748453, 4059.746737, 4079.744522, 4099.744430, 4119.741804, 4139.740704,
+ 4159.742056, 4179.740063, 4199.739155, 4219.741718, 4239.743433, 4259.743115, 4279.743265, 4299.745518,
+ 4319.746682, 4339.750491, 4359.748549, 4379.747226, 4399.750319, 4419.750379, 4439.752339, 4459.750561,
+ 4479.750612, 4499.747862, 4519.750693, 4539.747476, 4559.749051, 4579.750603, 4599.750089, 4619.754781,
+ 4639.755536, 4659.758879, 4679.762145, 4699.762199, 4719.763886, 4739.765081, 4759.765681, 4779.765427,
+ 4799.767582, 4819.769915, 4839.770107, 4859.769398, 4879.768976, 4899.768288, 4919.768786, 4939.768607,
+ 4959.772822, 4979.772833, 4999.775808, 5019.773468, 5039.776181, 5059.770860, 5079.773006, 5099.773136,
+ 5119.772474
+ },
+ // log K = 10
+ {
+ 737.833738, 757.236338, 776.997875, 797.119465, 817.602058, 838.445349, 859.650320, 881.216437,
+ 903.143414, 925.430957, 948.079220, 971.086231, 994.451449, 1018.173846, 1042.251928, 1066.683804,
+ 1091.467648, 1116.601416, 1142.083017, 1167.909918, 1194.079097, 1220.588887, 1247.435099, 1274.614860,
+ 1302.125360, 1329.962942, 1358.123177, 1386.601609, 1415.396299, 1444.501692, 1473.913256, 1503.628515,
+ 1533.642054, 1563.948888, 1594.544067, 1625.424305, 1656.582169, 1688.016543, 1719.718553, 1751.685990,
+ 1783.912130, 1816.393610, 1849.122123, 1882.095124, 1915.305647, 1948.748605, 1982.419843, 2016.313904,
+ 2050.422277, 2084.742618, 2119.270601, 2153.998538, 2188.920843, 2224.034736, 2259.333960, 2294.812594,
+ 2330.467059, 2366.291844, 2402.281356, 2438.428954, 2474.734330, 2511.187839, 2547.786319, 2584.527356,
+ 2621.406814, 2658.418551, 2695.554270, 2732.812564, 2770.190157, 2807.682744, 2845.287138, 2882.995251,
+ 2920.807982, 2958.719996, 2996.728771, 3034.826885, 3073.014082, 3111.287116, 3149.642093, 3188.073159,
+ 3226.577352, 3265.156836, 3303.803738, 3342.515111, 3381.291514, 3420.130671, 3459.024315, 3497.974631,
+ 3536.978221, 3576.034737, 3615.138610, 3654.285859, 3693.480455, 3732.715296, 3771.991792, 3811.305300,
+ 3850.654870, 3890.041152, 3929.457167, 3968.905021, 4008.381466, 4047.886976, 4087.419414, 4126.978997,
+ 4166.562624, 4206.166726, 4245.793284, 4285.445151, 4325.113012, 4364.802795, 4404.506274, 4444.228475,
+ 4483.965191, 4523.719569, 4563.487708, 4603.267460, 4643.060387, 4682.865430, 4722.680718, 4762.503248,
+ 4802.341326, 4842.185529, 4882.039088, 4921.900432, 4961.772625, 5001.649566, 5041.534380, 5081.425809,
+ 5121.323013, 5161.224908, 5201.130560, 5241.044625, 5280.962456, 5320.882373, 5360.810007, 5400.740058,
+ 5440.673843, 5480.606748, 5520.547394, 5560.493520, 5600.438100, 5640.383853, 5680.335386, 5720.292400,
+ 5760.250576, 5800.212705, 5840.172984, 5880.134991, 5920.097923, 5960.060294, 6000.031177, 6040.003398,
+ 6079.981696, 6119.954349, 6159.929074, 6199.908569, 6239.886163, 6279.863659, 6319.838261, 6359.815405,
+ 6399.795170, 6439.777225, 6479.755885, 6519.738857, 6559.723169, 6599.708549, 6639.695186, 6679.679788,
+ 6719.663289, 6759.651052, 6799.634129, 6839.624763, 6879.615013, 6919.605584, 6959.591310, 6999.586006,
+ 7039.577399, 7079.565236, 7119.557222, 7159.548481, 7199.533025, 7239.519648, 7279.511781, 7319.507639,
+ 7359.506105, 7399.497488, 7439.494944, 7479.491545, 7519.488415, 7559.480534, 7599.475646, 7639.470315,
+ 7679.468730, 7719.460928, 7759.455834, 7799.455686, 7839.451758, 7879.446841, 7919.443884, 7959.444138,
+ 7999.443141, 8039.446339, 8079.445285, 8119.444120, 8159.443157, 8199.446480, 8239.438893, 8279.437597,
+ 8319.439714, 8359.441566, 8399.440754, 8439.439156, 8479.440112, 8519.442365, 8559.444795, 8599.444497,
+ 8639.446714, 8679.449177, 8719.447620, 8759.443423, 8799.445995, 8839.446170, 8879.449622, 8919.450486,
+ 8959.450372, 8999.449731, 9039.448449, 9079.452579, 9119.451596, 9159.454020, 9199.456651, 9239.460902,
+ 9279.460419, 9319.461322, 9359.464611, 9399.467790, 9439.471448, 9479.468426, 9519.462982, 9559.464881,
+ 9599.462983, 9639.461706, 9679.457284, 9719.457023, 9759.455648, 9799.458786, 9839.456980, 9879.457411,
+ 9919.454971, 9959.451735, 9999.451916, 10039.443939, 10079.435325, 10119.432142, 10159.431313, 10199.431306,
+ 10239.429671
+ },
+ // log K = 11
+ {
+ 1476.444530, 1515.260638, 1554.794859, 1595.048761, 1636.023652, 1677.720065, 1720.138513, 1763.279158,
+ 1807.141823, 1851.724811, 1897.027530, 1943.048153, 1989.784818, 2037.234835, 2085.395149, 2134.263580,
+ 2183.834818, 2234.105548, 2285.071077, 2336.726064, 2389.065940, 2442.084276, 2495.776521, 2550.134177,
+ 2605.153465, 2660.825522, 2717.142881, 2774.098631, 2831.683790, 2889.889911, 2948.711233, 3008.137464,
+ 3068.158605, 3128.766926, 3189.951984, 3251.704129, 3314.013826, 3376.873018, 3440.270569, 3504.194851,
+ 3568.636295, 3633.586039, 3699.031630, 3764.962925, 3831.371336, 3898.245744, 3965.574384, 4033.345701,
+ 4101.550888, 4170.180686, 4239.221393, 4308.664143, 4378.497100, 4448.712735, 4519.296212, 4590.243535,
+ 4661.538747, 4733.174738, 4805.142002, 4877.427081, 4950.019513, 5022.915001, 5096.098561, 5169.567185,
+ 5243.312012, 5317.314578, 5391.572253, 5466.077564, 5540.821377, 5615.792025, 5690.985894, 5766.392174,
+ 5842.007403, 5917.816734, 5993.815607, 6069.999957, 6146.362662, 6222.895187, 6299.592875, 6376.440778,
+ 6453.443173, 6530.588182, 6607.874560, 6685.288536, 6762.829840, 6840.490840, 6918.272948, 6996.164556,
+ 7074.161694, 7152.259404, 7230.451098, 7308.737035, 7387.113244, 7465.575350, 7544.120230, 7622.733447,
+ 7701.427750, 7780.188044, 7859.016102, 7937.904850, 8016.853624, 8095.860379, 8174.919884, 8254.034842,
+ 8333.197794, 8412.405307, 8491.657429, 8570.946804, 8650.278289, 8729.648657, 8809.056098, 8888.496764,
+ 8967.966663, 9047.466051, 9126.993636, 9206.547289, 9286.127523, 9365.733732, 9445.361201, 9525.008810,
+ 9604.680107, 9684.363532, 9764.066435, 9843.787987, 9923.523567, 10003.272886, 10083.037524, 10162.811803,
+ 10242.597755, 10322.402415, 10402.216522, 10482.042584, 10561.877077, 10641.722187, 10721.570740, 10801.435848,
+ 10881.301249, 10961.178474, 11041.059765, 11120.951444, 11200.845806, 11280.746075, 11360.652145, 11440.567398,
+ 11520.476709, 11600.395391, 11680.316855, 11760.241900, 11840.170134, 11920.103548, 12000.038206, 12079.975425,
+ 12159.912459, 12239.857191, 12319.802744, 12399.754385, 12479.707055, 12559.658981, 12639.614004, 12719.566502,
+ 12799.526877, 12879.488833, 12959.451231, 13039.416480, 13119.388158, 13199.360789, 13279.334923, 13359.298927,
+ 13439.261641, 13519.235871, 13599.204919, 13679.181082, 13759.157391, 13839.137425, 13919.112545, 13999.096522,
+ 14079.082789, 14159.060487, 14239.040581, 14319.025236, 14399.006995, 14478.994078, 14558.980576, 14638.962774,
+ 14718.952192, 14798.938522, 14878.931501, 14958.919062, 15038.907891, 15118.897247, 15198.889129, 15278.881057,
+ 15358.873272, 15438.867627, 15518.861931, 15598.858206, 15678.850014, 15758.849382, 15838.847872, 15918.837658,
+ 15998.837920, 16078.831305, 16158.827345, 16238.815546, 16318.817302, 16398.814389, 16478.815820, 16558.811634,
+ 16638.808126, 16718.807170, 16798.805652, 16878.813389, 16958.806543, 17038.807885, 17118.813146, 17198.813752,
+ 17278.817574, 17358.820511, 17438.819443, 17518.814262, 17598.815511, 17678.814160, 17758.816726, 17838.817584,
+ 17918.814197, 17998.821276, 18078.822342, 18158.828474, 18238.822064, 18318.826732, 18398.827758, 18478.827319,
+ 18558.834153, 18638.835651, 18718.839105, 18798.840903, 18878.836463, 18958.836015, 19038.836351, 19118.832583,
+ 19198.829856, 19278.822953, 19358.824866, 19438.823697, 19518.826538, 19598.826954, 19678.831393, 19758.822279,
+ 19838.818757, 19918.808247, 19998.804516, 20078.809201, 20158.808900, 20238.806398, 20318.798516, 20398.802224,
+ 20478.803179
+ },
+ // log K = 12
+ {
+ 2953.666722, 3031.310137, 3110.389526, 3190.908699, 3272.865410, 3356.267166, 3441.114376, 3527.404284,
+ 3615.136846, 3704.310976, 3794.924283, 3886.973846, 3980.451903, 4075.355844, 4171.681168, 4269.423939,
+ 4368.569738, 4469.111760, 4571.043603, 4674.356370, 4779.038117, 4885.076002, 4992.460659, 5101.176875,
+ 5211.214098, 5322.553631, 5435.184633, 5549.088369, 5664.258175, 5780.673987, 5898.312682, 6017.158593,
+ 6137.195815, 6258.397951, 6380.761679, 6504.261630, 6628.870413, 6754.585867, 6881.374325, 7009.206926,
+ 7138.080376, 7267.979175, 7398.866805, 7530.722851, 7663.531580, 7797.269872, 7931.907340, 8067.451667,
+ 8203.855692, 8341.091306, 8479.155031, 8618.026003, 8757.682272, 8898.095612, 9039.253787, 9181.124777,
+ 9323.694329, 9466.944202, 9610.861899, 9755.403639, 9900.578346, 10046.356477, 10192.718770, 10339.635914,
+ 10487.081762, 10635.066232, 10783.560130, 10932.549371, 11082.024055, 11231.949071, 11382.332885, 11533.122831,
+ 11684.327555, 11835.935533, 11987.926999, 12140.281619, 12292.994375, 12446.046449, 12599.411083, 12753.088314,
+ 12907.079745, 13061.355923, 13215.895621, 13370.722085, 13525.795736, 13681.095971, 13836.645692, 13992.405906,
+ 14148.380716, 14304.566860, 14460.959382, 14617.539754, 14774.296125, 14931.214508, 15088.271748, 15245.493870,
+ 15402.867309, 15560.380994, 15718.035423, 15875.801492, 16033.687891, 16191.701231, 16349.808763, 16508.026001,
+ 16666.336068, 16824.745049, 16983.259302, 17141.848044, 17300.504517, 17459.240272, 17618.060296, 17776.914688,
+ 17935.848010, 18094.852645, 18253.914288, 18413.023567, 18572.168358, 18731.376016, 18890.631505, 19049.948196,
+ 19209.279260, 19368.638871, 19528.043860, 19687.498901, 19846.941765, 20006.443084, 20165.973535, 20325.525957,
+ 20485.109520, 20644.729917, 20804.363876, 20964.021778, 21123.698687, 21283.379344, 21443.080536, 21602.821439,
+ 21762.568135, 21922.305338, 22082.065878, 22241.856145, 22401.629436, 22561.435266, 22721.262644, 22881.080478,
+ 23040.916036, 23200.734053, 23360.583255, 23520.433188, 23680.275065, 23840.132678, 24000.012847, 24159.886135,
+ 24319.788338, 24479.654024, 24639.540316, 24799.466449, 24959.369335, 25119.276072, 25279.186502, 25439.091982,
+ 25599.020083, 25758.930671, 25918.847522, 26078.773060, 26238.704021, 26398.647566, 26558.603366, 26718.539855,
+ 26878.464425, 27038.402892, 27198.344989, 27358.270906, 27518.203448, 27678.169545, 27838.133625, 27998.111630,
+ 28158.064264, 28318.032491, 28477.994153, 28637.950838, 28797.910362, 28957.884994, 29117.861910, 29277.814384,
+ 29437.774223, 29597.762256, 29757.729819, 29917.688497, 30077.675371, 30237.667941, 30397.639431, 30557.644527,
+ 30717.622991, 30877.622641, 31037.598941, 31197.601527, 31357.599118, 31517.585221, 31677.569983, 31837.556138,
+ 31997.527880, 32157.500622, 32317.491709, 32477.504936, 32637.501123, 32797.516330, 32957.547336, 33117.541765,
+ 33277.561207, 33437.554332, 33597.564988, 33757.565321, 33917.561180, 34077.580598, 34237.597132, 34397.598246,
+ 34557.617536, 34717.623318, 34877.649722, 35037.645029, 35197.681143, 35357.700478, 35517.701804, 35677.734383,
+ 35837.736494, 35997.737103, 36157.732045, 36317.741539, 36477.735602, 36637.761491, 36797.742508, 36957.734137,
+ 37117.742952, 37277.789218, 37437.793798, 37597.779066, 37757.796449, 37917.794116, 38077.804399, 38237.797507,
+ 38397.788049, 38557.768583, 38717.768279, 38877.752553, 39037.747349, 39197.737646, 39357.774395, 39517.752643,
+ 39677.753347, 39837.740038, 39997.719486, 40157.709578, 40317.708657, 40477.719261, 40637.726978, 40797.711139,
+ 40957.707784
+ },
+ // log K = 13
+ {
+ 5908.111420, 6063.410343, 6221.578483, 6382.623803, 6546.553966, 6713.369171, 6883.070010, 7055.656719,
+ 7231.129758, 7409.486672, 7590.720089, 7774.822139, 7961.786850, 8151.601206, 8344.261218, 8539.750086,
+ 8738.048786, 8939.141702, 9143.011205, 9349.645220, 9559.003557, 9771.078383, 9985.846791, 10203.277103,
+ 10423.349219, 10646.031323, 10871.293001, 11099.111735, 11329.434411, 11562.255804, 11797.522109, 12035.204106,
+ 12275.263649, 12517.670237, 12762.388671, 13009.375491, 13258.587519, 13510.006825, 13763.564399, 14019.227089,
+ 14276.975295, 14536.745792, 14798.511796, 15062.215057, 15327.806593, 15595.257499, 15864.529774, 16135.586770,
+ 16408.362074, 16682.849583, 16958.977019, 17236.708221, 17515.990972, 17796.798914, 18079.096585, 18362.834517,
+ 18647.959152, 18934.452073, 19222.251998, 19511.366053, 19801.717928, 20093.269020, 20385.960319, 20679.772736,
+ 20974.698498, 21270.684814, 21567.671010, 21865.649193, 22164.581341, 22464.423293, 22765.172894, 23066.785452,
+ 23369.201089, 23672.387494, 23976.354000, 24281.072155, 24586.465328, 24892.557323, 25199.303712, 25506.654130,
+ 25814.626835, 26123.148009, 26432.247046, 26741.889610, 27052.042135, 27362.685181, 27673.761784, 27985.292246,
+ 28297.219809, 28609.572368, 28922.317633, 29235.448741, 29548.944253, 29862.766175, 30176.910610, 30491.363107,
+ 30806.103787, 31121.126024, 31436.412153, 31751.975134, 32067.737444, 32383.768560, 32700.017044, 33016.458300,
+ 33333.101934, 33649.904492, 33966.923435, 34284.105041, 34601.429152, 34918.876076, 35236.481998, 35554.235511,
+ 35872.093847, 36190.101572, 36508.190892, 36826.420800, 37144.739624, 37463.138566, 37781.598984, 38100.191087,
+ 38418.846511, 38737.587005, 39056.391321, 39375.255819, 39694.182864, 40013.173503, 40332.221146, 40651.311460,
+ 40970.465534, 41289.697002, 41608.913381, 41928.242461, 42247.585089, 42566.988678, 42886.409594, 43205.849396,
+ 43525.322840, 43844.784209, 44164.327769, 44483.869861, 44803.383648, 45122.975593, 45442.593765, 45762.258110,
+ 46081.937012, 46401.605265, 46721.295022, 47041.004034, 47360.695978, 47680.435451, 48000.192231, 48319.953839,
+ 48639.726508, 48959.506239, 49279.318744, 49599.117846, 49918.951926, 50238.772730, 50558.620076, 50878.455501,
+ 51198.285377, 51518.135316, 51837.976134, 52157.809260, 52477.635805, 52797.492063, 53117.368983, 53437.262930,
+ 53757.142921, 54077.026859, 54396.907345, 54716.835955, 55036.713062, 55356.640827, 55676.537369, 55996.488865,
+ 56316.420754, 56636.355217, 56956.329593, 57276.245282, 57596.202473, 57916.159021, 58236.077854, 58556.010303,
+ 58875.938110, 59195.924442, 59515.871947, 59835.838035, 60155.809948, 60475.808167, 60795.765545, 61115.766398,
+ 61435.725825, 61755.677959, 62075.688445, 62395.647263, 62715.609853, 63035.591787, 63355.588060, 63675.579841,
+ 63995.560980, 64315.552298, 64635.582956, 64955.538779, 65275.512154, 65595.490993, 65915.477332, 66235.470693,
+ 66555.483426, 66875.455452, 67195.455039, 67515.435090, 67835.418300, 68155.401870, 68475.404599, 68795.452423,
+ 69115.474466, 69435.450344, 69755.439668, 70075.479966, 70395.468204, 70715.475957, 71035.472807, 71355.513734,
+ 71675.486047, 71995.450475, 72315.443410, 72635.425138, 72955.481520, 73275.482426, 73595.476688, 73915.469540,
+ 74235.426693, 74555.381683, 74875.368981, 75195.429157, 75515.480428, 75835.517265, 76155.557945, 76475.556134,
+ 76795.579451, 77115.605924, 77435.604178, 77755.572770, 78075.576305, 78395.562526, 78715.547599, 79035.551674,
+ 79355.540615, 79675.575726, 79995.547586, 80315.544278, 80635.546227, 80955.555617, 81275.540591, 81595.549538,
+ 81915.550079
+ },
+ // log K = 14
+ {
+ 11817.000969, 12127.609164, 12443.958867, 12766.059919, 13093.926347, 13427.563687, 13766.972021, 14112.158775,
+ 14463.118632, 14819.833221, 15182.310838, 15550.514164, 15924.452624, 16304.097682, 16689.414687, 17080.390156,
+ 17476.985675, 17879.171048, 18286.913146, 18700.171223, 19118.909859, 19543.074108, 19972.606221, 20407.466793,
+ 20847.600290, 21292.965548, 21743.489087, 22199.109276, 22659.760749, 23125.379711, 23595.914325, 24071.277989,
+ 24551.407611, 25036.233071, 25525.671155, 26019.641791, 26518.074136, 27020.885189, 27528.004081, 28039.347478,
+ 28554.818908, 29074.343878, 29597.869471, 30125.254970, 30656.449891, 31191.351228, 31729.908970, 32272.003587,
+ 32817.586623, 33366.531359, 33918.769390, 34474.232727, 35032.798601, 35594.423192, 36159.024745, 36726.469786,
+ 37296.712851, 37869.686035, 38445.278953, 39023.442285, 39604.084062, 40187.147209, 40772.503600, 41360.148758,
+ 41949.978377, 42541.929077, 43135.892496, 43731.836459, 44329.664552, 44929.334224, 45530.772542, 46133.919152,
+ 46738.746702, 47345.120215, 47953.051656, 48562.469011, 49173.281212, 49785.453501, 50398.908717, 51013.645932,
+ 51629.563526, 52246.635759, 52864.820434, 53484.057447, 54104.303719, 54725.504675, 55347.680019, 55970.744363,
+ 56594.688890, 57219.458967, 57844.949821, 58471.197998, 59098.180259, 59725.774793, 60354.058215, 60982.962558,
+ 61612.427469, 62242.448368, 62873.013426, 63504.106743, 64135.672742, 64767.665213, 65400.082326, 66032.949331,
+ 66666.219558, 67299.817447, 67933.801373, 68568.092593, 69202.717636, 69837.643507, 70472.852133, 71108.305774,
+ 71744.023189, 72380.016332, 73016.168118, 73652.560876, 74289.210741, 74926.041959, 75563.048434, 76200.207724,
+ 76837.505542, 77474.999283, 78112.608284, 78750.375633, 79388.254423, 80026.267074, 80664.419197, 81302.602941,
+ 81940.909776, 82579.321050, 83217.801329, 83856.420605, 84495.098999, 85133.832545, 85772.588001, 86411.448347,
+ 87050.399262, 87689.358100, 88328.463714, 88967.516990, 89606.618839, 90245.788934, 90884.986443, 91524.243962,
+ 92163.523370, 92802.835394, 93442.198790, 94081.633840, 94721.043838, 95360.482304, 96000.002974, 96639.518339,
+ 97279.069377, 97918.584802, 98558.189888, 99197.773242, 99837.390001, 100476.994827, 101116.655652, 101756.293703,
+ 102395.977437, 103035.662337, 103675.360919, 104315.063845, 104954.783621, 105594.504004, 106234.229823, 106874.056079,
+ 107513.846129, 108153.562857, 108793.393622, 109433.202056, 110072.966808, 110712.777026, 111352.577741, 111992.441538,
+ 112632.256702, 113272.083743, 113911.943484, 114551.818432, 115191.687411, 115831.584194, 116471.522453, 117111.338979,
+ 117751.192777, 118391.070556, 119030.991702, 119670.888488, 120310.801509, 120950.761000, 121590.691826, 122230.618394,
+ 122870.618097, 123510.616056, 124150.552363, 124790.488946, 125430.445483, 126070.428297, 126710.409643, 127350.398562,
+ 127990.397846, 128630.413222, 129270.354937, 129910.365912, 130550.332174, 131190.259646, 131830.280970, 132470.304278,
+ 133110.295205, 133750.312426, 134390.364847, 135030.321000, 135670.319926, 136310.335570, 136950.324790, 137590.338289,
+ 138230.368657, 138870.374084, 139510.447896, 140150.468353, 140790.441338, 141430.438424, 142070.440787, 142710.431348,
+ 143350.433770, 143990.493350, 144630.548409, 145270.549935, 145910.530486, 146550.488808, 147190.510290, 147830.509426,
+ 148470.575641, 149110.541845, 149750.592427, 150390.634921, 151030.706500, 151670.646574, 152310.624482, 152950.686127,
+ 153590.659153, 154230.646236, 154870.720079, 155510.614028, 156150.596142, 156790.659641, 157430.633386, 158070.649047,
+ 158710.591350, 159350.585980, 159990.557711, 160630.530183, 161270.499427, 161910.521323, 162550.516603, 163190.482792,
+ 163830.422631
+ },
+ // log K = 15
+ {
+ 23634.780143, 24256.008404, 24888.717596, 25532.934303, 26188.680791, 26855.965949, 27534.798395, 28225.179199,
+ 28927.099457, 29640.545847, 30365.501013, 31101.936533, 31849.825685, 32609.111850, 33379.755958, 34161.706474,
+ 34954.912131, 35759.291030, 36574.778310, 37401.280667, 38238.742234, 39087.060449, 39946.133936, 40815.861824,
+ 41696.151280, 42586.877221, 43487.927203, 44399.169293, 45320.460825, 46251.693279, 47192.741847, 48143.464216,
+ 49103.716679, 50073.349113, 51052.208336, 52040.138165, 53036.993192, 54042.604241, 55056.838449, 56079.501709,
+ 57110.434199, 58149.484623, 59196.455710, 60251.226284, 61313.593108, 62383.385277, 63460.484960, 64544.666337,
+ 65635.763495, 66733.629361, 67838.096489, 68948.987037, 70066.117363, 71189.338133, 72318.464573, 73453.423815,
+ 74593.923438, 75739.864335, 76891.058098, 78047.391508, 79208.654009, 80374.753680, 81545.514022, 82720.817799,
+ 83900.405263, 85084.273208, 86272.263471, 87464.134340, 88659.829437, 89859.204804, 91062.115479, 92268.425863,
+ 93478.041907, 94690.804589, 95906.647693, 97125.398400, 98347.018145, 99571.305992, 100798.274503, 102027.737913,
+ 103259.574749, 104493.727918, 105730.035415, 106968.465700, 108208.988306, 109451.445144, 110695.748927, 111941.827269,
+ 113189.666448, 114439.093665, 115690.116530, 116942.608958, 118196.501294, 119451.722652, 120708.276299, 121966.046247,
+ 123225.027480, 124485.082349, 125746.201609, 127008.357433, 128271.416054, 129535.417336, 130800.297278, 132066.008820,
+ 133332.483576, 134599.733640, 135867.681429, 137136.303792, 138405.538969, 139675.413251, 140945.839375, 142216.787412,
+ 143488.232199, 144760.234976, 146032.650196, 147305.459198, 148578.703708, 149852.356585, 151126.347770, 152400.641475,
+ 153675.275168, 154950.179904, 156225.390849, 157500.864126, 158776.585926, 160052.504602, 161328.716398, 162605.171897,
+ 163881.825931, 165158.677102, 166435.646121, 167712.814846, 168990.136047, 170267.619890, 171545.238277, 172823.011460,
+ 174100.890250, 175378.821293, 176656.920467, 177935.093547, 179213.318173, 180491.692642, 181770.130424, 183048.727206,
+ 184327.217835, 185605.921891, 186884.657773, 188163.418059, 189442.221838, 190721.122201, 192000.068271, 193279.087882,
+ 194558.139554, 195837.195523, 197116.400538, 198395.612565, 199674.831001, 200954.101970, 202233.365695, 203512.639968,
+ 204792.003319, 206071.379608, 207350.730914, 208630.080321, 209909.544216, 211189.014497, 212468.508049, 213748.055445,
+ 215027.584964, 216307.216078, 217586.896296, 218866.477232, 220146.109506, 221425.767458, 222705.465633, 223985.129542,
+ 225264.797531, 226544.544335, 227824.304232, 229104.031191, 230383.739676, 231663.424684, 232943.198829, 234222.906969,
+ 235502.669185, 236782.475696, 238062.262723, 239342.090234, 240622.002668, 241901.911022, 243181.742902, 244461.690551,
+ 245741.531359, 247021.432644, 248301.277604, 249581.168404, 250861.094078, 252141.038818, 253421.065594, 254701.020169,
+ 255980.887518, 257260.848328, 258540.873565, 259820.905666, 261100.908594, 262380.854763, 263660.862699, 264940.834842,
+ 266220.775656, 267500.745402, 268780.761926, 270060.823540, 271340.798715, 272620.823081, 273900.881190, 275180.928531,
+ 276460.914606, 277740.919081, 279020.967358, 280301.002010, 281581.054314, 282861.097638, 284141.109728, 285421.181335,
+ 286701.182750, 287981.231437, 289261.342952, 290541.328315, 291821.320901, 293101.395969, 294381.402286, 295661.374812,
+ 296941.357112, 298221.462446, 299501.443884, 300781.429735, 302061.512800, 303341.459364, 304621.530042, 305901.507763,
+ 307181.541613, 308461.506346, 309741.474814, 311021.458944, 312301.373851, 313581.258984, 314861.222655, 316141.192042,
+ 317421.179698, 318701.162346, 319981.143104, 321261.185706, 322541.120379, 323821.065555, 325100.961639, 326380.920540,
+ 327660.786948
+ },
+ // log K = 16
+ {
+ 47270.338530, 48512.804588, 49778.235006, 51066.679846, 52378.178287, 53712.756179, 55070.430016, 56451.198432,
+ 57855.044557, 59281.941569, 60731.855114, 62204.727445, 63700.498656, 65219.087806, 66760.381754, 68324.278827,
+ 69910.681844, 71519.442760, 73150.426782, 74803.469853, 76478.400456, 78175.038414, 79893.184924, 81632.639705,
+ 83393.214587, 85174.662642, 86976.739912, 88799.210643, 90641.829434, 92504.355605, 94386.478165, 96287.925119,
+ 98208.422967, 100147.694840, 102105.387259, 104081.226364, 106074.916897, 108086.136683, 110114.560800, 112159.857579,
+ 114221.724675, 116299.785348, 118393.731853, 120503.258946, 122627.958417, 124767.540962, 126921.668848, 129090.003274,
+ 131272.204811, 133467.918004, 135676.870426, 137898.595565, 140132.829694, 142379.269230, 144637.536478, 146907.321805,
+ 149188.299797, 151480.151490, 153782.571459, 156095.292246, 158417.834407, 160749.983694, 163091.465145, 165441.984077,
+ 167801.236994, 170168.903670, 172544.767987, 174928.494171, 177319.918928, 179718.624717, 182124.426670, 184537.056818,
+ 186956.260737, 189381.869001, 191813.551321, 194251.107949, 196694.280093, 199142.910671, 201596.663557, 204055.453662,
+ 206519.090402, 208987.322132, 211460.079506, 213936.962995, 216417.945039, 218902.907481, 221391.594168, 223883.857806,
+ 226379.519239, 228878.478995, 231380.515508, 233885.422831, 236393.251620, 238903.734661, 241416.885791, 243932.462444,
+ 246450.361381, 248970.497360, 251492.768859, 254017.059221, 256543.244066, 259071.366198, 261601.098326, 264132.472390,
+ 266665.477130, 269200.003838, 271735.898262, 274273.128446, 276811.699617, 279351.334667, 281892.176977, 284434.126935,
+ 286977.141855, 289520.993664, 292065.816286, 294611.452586, 297157.986206, 299705.252956, 302253.226506, 304801.859484,
+ 307351.170990, 309901.148395, 312451.594894, 315002.530644, 317553.961670, 320105.951895, 322658.553440, 325211.299379,
+ 327764.626147, 330318.311300, 332872.331048, 335426.696106, 337981.358498, 340536.275072, 343091.446562, 345646.972942,
+ 348202.757347, 350758.693296, 353314.808579, 355871.203754, 358427.702337, 360984.491079, 363541.449032, 366098.493281,
+ 368655.687887, 371213.030472, 373770.561193, 376328.233445, 378886.001706, 381443.937538, 384001.856504, 386559.857503,
+ 389117.926075, 391676.219835, 394234.530250, 396792.812601, 399351.291660, 401909.794770, 404468.330377, 407026.964781,
+ 409585.614496, 412144.407471, 414703.175858, 417261.980659, 419820.958264, 422379.863083, 424938.751186, 427497.927620,
+ 430056.993719, 432616.066348, 435175.266190, 437734.382772, 440293.531277, 442852.711332, 445412.016423, 447971.502143,
+ 450530.798765, 453090.167199, 455649.502729, 458208.951251, 460768.459694, 463328.053373, 465887.570627, 468447.239271,
+ 471006.792818, 473566.544068, 476126.194227, 478685.887605, 481245.577573, 483805.348711, 486365.028302, 488924.844568,
+ 491484.612666, 494044.321666, 496604.011864, 499163.799789, 501723.629793, 504283.493711, 506843.290195, 509403.157856,
+ 511962.990320, 514522.941350, 517082.791953, 519642.848656, 522202.833761, 524762.686522, 527322.632758, 529882.529037,
+ 532442.420851, 535002.422317, 537562.386309, 540122.357425, 542682.434171, 545242.468829, 547802.466694, 550362.492718,
+ 552922.461624, 555482.539495, 558042.531431, 560602.467437, 563162.648952, 565722.750610, 568282.702877, 570842.810092,
+ 573402.917604, 575962.999235, 578523.081937, 581083.162584, 583643.265455, 586203.287883, 588763.260931, 591323.292027,
+ 593883.252456, 596443.273935, 599003.353442, 601563.492606, 604123.421135, 606683.438234, 609243.605174, 611803.638038,
+ 614363.628593, 616923.690580, 619483.675256, 622043.744171, 624603.673246, 627163.577236, 629723.677960, 632283.670157,
+ 634843.692958, 637403.576097, 639963.511752, 642523.482942, 645083.419327, 647643.330711, 650203.272467, 652763.290376,
+ 655323.267522
+ },
+ // log K = 17
+ {
+ 94541.455324, 97026.403245, 99557.280165, 102134.178452, 104757.187033, 107426.362674, 110141.726029, 112903.278005,
+ 115710.975458, 118564.785344, 121464.629896, 124410.377345, 127401.910279, 130439.087070, 133521.683723, 136649.499031,
+ 139822.321627, 143039.848320, 146301.785984, 149607.844773, 152957.695333, 156350.945326, 159787.233040, 163266.157476,
+ 166787.301371, 170350.201319, 173954.353438, 177599.302692, 181284.526367, 185009.507089, 188773.757457, 192576.638207,
+ 196417.619925, 200296.089540, 204211.494530, 208163.221220, 212150.613086, 216173.056150, 220229.879308, 224320.462264,
+ 228444.146459, 232600.284426, 236788.187347, 241007.264352, 245256.725247, 249535.938618, 253844.217048, 258180.876528,
+ 262545.282090, 266936.686787, 271354.453768, 275797.958112, 280266.429404, 284759.298770, 289275.861948, 293815.386243,
+ 298377.390708, 302961.087179, 307565.895452, 312191.173759, 316836.362522, 321500.721532, 326183.652758, 330884.723656,
+ 335603.197219, 340338.617624, 345090.316855, 349857.860719, 354640.538362, 359438.001710, 364249.638745, 369074.940533,
+ 373913.351331, 378764.438864, 383627.767868, 388502.854215, 393389.259584, 398286.483436, 403194.238263, 408111.948151,
+ 413039.292303, 417975.909674, 422921.304084, 427875.195485, 432837.200959, 437807.062429, 442784.399966, 447768.726662,
+ 452760.032525, 457757.794185, 462761.810279, 467771.737196, 472787.226777, 477808.283249, 482834.411114, 487865.521166,
+ 492901.275392, 497941.616150, 502986.164281, 508034.749030, 513087.210697, 518143.298172, 523202.692981, 528265.454984,
+ 533331.384435, 538400.437943, 543472.278711, 548546.790172, 553623.737338, 558703.047575, 563784.666463, 568868.524932,
+ 573954.251207, 579041.935016, 584131.557779, 589222.910016, 594315.786547, 599410.314112, 604506.227555, 609603.610432,
+ 614702.124262, 619801.918486, 624902.903258, 630005.085347, 635107.990499, 640211.847431, 645316.755403, 650422.424613,
+ 655529.009211, 660636.166872, 665744.170495, 670852.806947, 675962.099161, 681072.042796, 686182.458370, 691293.269596,
+ 696404.763335, 701516.635253, 706628.965299, 711741.575313, 716854.769083, 721968.220605, 727082.027784, 732196.114131,
+ 737310.594653, 742425.339665, 747540.295793, 752655.505647, 757770.968129, 762886.636891, 768002.527720, 773118.559142,
+ 778234.871441, 783351.273881, 788467.944809, 793584.599211, 798701.497594, 803818.484285, 808935.619316, 814053.067243,
+ 819170.390229, 824287.946609, 829405.450988, 834523.054071, 839640.856519, 844758.800102, 849876.719924, 854994.831607,
+ 860112.896788, 865231.116455, 870349.453898, 875467.910756, 880586.444976, 885705.102070, 890823.822978, 895942.513436,
+ 901061.120048, 906179.716020, 911298.587548, 916417.583647, 921536.507373, 926655.570709, 931774.690027, 936893.879961,
+ 942012.956398, 947132.142141, 952251.503479, 957370.815448, 962490.012685, 967609.521437, 972729.002837, 977848.339983,
+ 982967.824448, 988087.392216, 993206.992691, 998326.635354, 1003446.304388, 1008565.985024, 1013685.706678, 1018805.581548,
+ 1023925.239073, 1029045.157502, 1034165.035124, 1039284.843524, 1044404.676526, 1049524.620186, 1054644.441170, 1059764.409794,
+ 1064884.318434, 1070004.338968, 1075124.268749, 1080244.471055, 1085364.528193, 1090484.511927, 1095604.595114, 1100724.598478,
+ 1105844.666500, 1110964.804168, 1116084.864159, 1121204.874577, 1126325.062599, 1131445.152321, 1136565.301592, 1141685.638897,
+ 1146805.806800, 1151925.790859, 1157045.932640, 1162166.112443, 1167286.221698, 1172406.273611, 1177526.581875, 1182646.832239,
+ 1187766.882661, 1192886.884388, 1198006.962839, 1203127.084377, 1208246.959060, 1213366.976076, 1218486.763464, 1223606.758568,
+ 1228726.721761, 1233846.593404, 1238966.815228, 1244086.865449, 1249206.854850, 1254326.996667, 1259447.092239, 1264567.198081,
+ 1269687.499091, 1274807.474194, 1279927.244161, 1285047.273317, 1290167.090035, 1295287.035620, 1300406.888764, 1305526.667946,
+ 1310646.540340
+ },
+ // log K = 18
+ {
+ 189083.688921, 194053.597973, 199115.342781, 204269.157017, 209515.195566, 214853.553887, 220284.295855, 225807.399998,
+ 231422.826900, 237130.469824, 242930.139690, 248821.634191, 254804.718978, 260879.073467, 267044.289594, 273299.936993,
+ 279645.549742, 286080.586199, 292604.521050, 299216.651567, 305916.376160, 312702.908374, 319575.536730, 326533.420485,
+ 333575.639430, 340701.388339, 347909.713977, 355199.642852, 362570.108280, 370020.154080, 377548.652175, 385154.413997,
+ 392836.374125, 400593.371525, 408424.172521, 416327.671123, 424302.406749, 432347.292674, 440461.014310, 448642.241683,
+ 456889.603709, 465201.946081, 473577.749277, 482015.802141, 490514.770444, 499073.243760, 507689.794308, 516363.149230,
+ 525091.902823, 533874.850510, 542710.434311, 551597.433198, 560534.241273, 569520.019389, 578553.034265, 587632.180111,
+ 596756.084542, 605923.440522, 615133.037840, 624383.626242, 633673.841133, 643002.674316, 652368.578283, 661770.556501,
+ 671207.605795, 680678.417768, 690181.818697, 699716.780471, 709282.084973, 718876.864175, 728499.951143, 738150.280049,
+ 747827.053968, 757529.236597, 767255.784062, 777006.100829, 786779.065927, 796573.507429, 806388.828153, 816223.976258,
+ 826078.608350, 835951.751956, 845842.493689, 855750.225870, 865674.190527, 875613.809349, 885568.425527, 895537.272219,
+ 905519.763307, 915515.310346, 925523.353103, 935543.172648, 945574.292664, 955616.280042, 965668.428231, 975730.535479,
+ 985802.055447, 995882.596628, 1005971.583108, 1016068.664992, 1026173.649961, 1036285.774891, 1046404.955770, 1056530.588186,
+ 1066662.648243, 1076800.535542, 1086944.269881, 1097093.058652, 1107247.076183, 1117405.990557, 1127569.310917, 1137737.011345,
+ 1147908.717700, 1158084.362020, 1168263.501029, 1178446.221907, 1188632.206842, 1198821.131431, 1209013.020373, 1219207.570786,
+ 1229404.854893, 1239604.423801, 1249806.433263, 1260010.619993, 1270216.819018, 1280424.774173, 1290634.560600, 1300846.097203,
+ 1311059.128630, 1321273.628402, 1331489.832488, 1341707.300774, 1351925.947953, 1362145.802925, 1372366.802414, 1382588.547896,
+ 1392811.264776, 1403035.227449, 1413259.922280, 1423485.322819, 1433711.916106, 1443938.981360, 1454166.596984, 1464394.761679,
+ 1474623.684212, 1484853.156422, 1495083.020146, 1505313.735890, 1515544.731764, 1525776.086634, 1536007.742073, 1546239.679012,
+ 1556472.212370, 1566704.988763, 1576937.965495, 1587171.446007, 1597405.292186, 1607639.395274, 1617873.660775, 1628108.037273,
+ 1638342.741799, 1648577.832828, 1658813.325573, 1669048.803784, 1679284.679963, 1689520.927615, 1699757.050010, 1709993.263665,
+ 1720229.557663, 1730466.102090, 1740702.604009, 1750939.618504, 1761176.930182, 1771414.210528, 1781651.713742, 1791889.218503,
+ 1802126.805234, 1812364.257333, 1822602.054007, 1832839.687792, 1843077.871973, 1853315.944762, 1863554.202066, 1873792.633125,
+ 1884030.883830, 1894269.182711, 1904507.712050, 1914746.469507, 1924985.484753, 1935224.245956, 1945463.225215, 1955702.268883,
+ 1965941.321722, 1976180.037370, 1986419.052784, 1996658.429579, 2006897.450283, 2017136.904971, 2027376.240039, 2037615.549295,
+ 2047855.197425, 2058094.765906, 2068334.279787, 2078573.820075, 2088813.469730, 2099053.348620, 2109293.174176, 2119533.012704,
+ 2129772.820132, 2140012.744709, 2150252.607739, 2160492.533017, 2170732.406422, 2180972.400110, 2191212.436319, 2201452.637894,
+ 2211692.339292, 2221932.460057, 2232172.746036, 2242412.892705, 2252653.174824, 2262893.432271, 2273134.002270, 2283374.311008,
+ 2293614.537881, 2303854.703732, 2314095.191374, 2324335.236186, 2334575.624526, 2344816.313269, 2355056.224171, 2365296.361285,
+ 2375537.048184, 2385777.280284, 2396017.222502, 2406257.145546, 2416497.272190, 2426737.212210, 2436977.500856, 2447217.897426,
+ 2457458.423605, 2467698.536697, 2477938.448096, 2488178.544726, 2498418.640441, 2508658.160374, 2518898.274852, 2529138.509345,
+ 2539378.310538, 2549617.989497, 2559857.504886, 2570097.363128, 2580337.195261, 2590576.878833, 2600816.572159, 2611055.930087,
+ 2621295.625004
+ },
+ // log K = 19
+ {
+ 378168.156126, 388107.986863, 398231.517433, 408539.178530, 419031.252008, 429707.954086, 440569.365328, 451615.642722,
+ 462846.490174, 474261.790112, 485861.190310, 497644.259253, 509610.438730, 521759.095095, 534089.573442, 546600.981190,
+ 559292.272697, 572162.409622, 585210.331052, 598434.669889, 611834.157416, 625407.271721, 639152.543961, 653068.262294,
+ 667152.777393, 681404.241581, 695820.778241, 710400.501586, 725141.425404, 740041.558661, 755098.503731, 770310.107475,
+ 785674.192935, 801188.297422, 816849.950298, 832656.623960, 848606.198026, 864696.151343, 880923.487841, 897285.841319,
+ 913780.530043, 930405.131973, 947156.655545, 964032.974599, 981030.762278, 998147.499138, 1015380.659035, 1032727.149847,
+ 1050184.616751, 1067750.330789, 1085421.716664, 1103195.771853, 1121069.796650, 1139041.324454, 1157107.373802, 1175265.377022,
+ 1193513.201704, 1211847.872855, 1230267.133114, 1248768.293478, 1267348.634472, 1286005.987451, 1304738.187318, 1323542.421988,
+ 1342416.458954, 1361357.807871, 1380364.530293, 1399434.095493, 1418564.587724, 1437754.116682, 1457000.254889, 1476301.275902,
+ 1495654.977143, 1515059.332212, 1534512.821906, 1554013.130304, 1573558.585534, 1593147.244450, 1612777.978405, 1632448.600136,
+ 1652157.704100, 1671904.308928, 1691686.012962, 1711501.716352, 1731349.849417, 1751229.053668, 1771138.423208, 1791076.196899,
+ 1811041.338260, 1831032.177526, 1851048.091817, 1871088.096683, 1891150.317142, 1911234.619162, 1931339.423184, 1951463.947205,
+ 1971606.994543, 1991767.606441, 2011945.852451, 2032140.098239, 2052349.709954, 2072574.274980, 2092812.387905, 2113063.537955,
+ 2133327.750182, 2153603.787963, 2173890.933053, 2194188.225246, 2214496.333997, 2234814.120065, 2255140.753372, 2275476.675949,
+ 2295820.689602, 2316172.220323, 2336530.258616, 2356894.932256, 2377266.633368, 2397645.173978, 2418028.419639, 2438417.586475,
+ 2458812.422593, 2479211.306360, 2499615.253217, 2520023.375080, 2540436.051898, 2560851.967345, 2581271.569076, 2601694.318944,
+ 2622120.693963, 2642549.418269, 2662981.070023, 2683415.520909, 2703852.500003, 2724291.876933, 2744733.213126, 2765177.477758,
+ 2785623.320019, 2806070.985142, 2826520.234343, 2846971.203929, 2867423.303616, 2887876.396540, 2908331.739769, 2928788.290672,
+ 2949245.276635, 2969704.620226, 2990164.657583, 3010625.841710, 3031087.855626, 3051550.619090, 3072014.642350, 3092479.163042,
+ 3112944.278087, 3133410.101603, 3153876.719100, 3174343.414117, 3194810.517970, 3215278.762802, 3235747.174277, 3256216.390486,
+ 3276685.652553, 3297155.394593, 3317625.911419, 3338096.547090, 3358567.567862, 3379038.637659, 3399510.649493, 3419983.382105,
+ 3440456.231853, 3460929.496047, 3481403.703904, 3501877.874941, 3522351.898403, 3542825.527074, 3563300.038829, 3583775.102749,
+ 3604249.896903, 3624724.967423, 3645201.027293, 3665676.534460, 3686152.061202, 3706628.308333, 3727104.271827, 3747580.803335,
+ 3768057.710876, 3788534.660048, 3809011.720864, 3829489.164523, 3849967.204462, 3870445.043912, 3890923.080446, 3911400.767281,
+ 3931879.061622, 3952357.218696, 3972836.284463, 3993314.516935, 4013793.278282, 4034272.661280, 4054751.814018, 4075230.503334,
+ 4095708.557968, 4116187.919814, 4136667.714093, 4157147.152355, 4177627.302553, 4198107.383728, 4218587.174918, 4239066.631662,
+ 4259545.994167, 4280026.100110, 4300506.380609, 4320986.617208, 4341466.696312, 4361945.995732, 4382426.527708, 4402907.058309,
+ 4423386.556967, 4443867.498542, 4464348.284664, 4484828.344391, 4505309.026083, 4525789.774619, 4546270.396129, 4566750.818685,
+ 4587231.362126, 4607711.356124, 4628191.883250, 4648673.308942, 4669153.953372, 4689634.118530, 4710114.637510, 4730595.217429,
+ 4751075.358693, 4771555.492921, 4792035.313408, 4812516.057037, 4832995.761283, 4853475.977619, 4873956.732160, 4894436.879314,
+ 4914917.149285, 4935397.069776, 4955877.550351, 4976357.852728, 4996838.246842, 5017318.023169, 5037798.008344, 5058278.290403,
+ 5078757.968956, 5099237.419344, 5119716.759591, 5140195.935830, 5160675.636320, 5181154.980843, 5201633.780713, 5222113.031789,
+ 5242591.881577
+ },
+ // log K = 20
+ {
+ 756337.090537, 776216.744995, 796463.785646, 817079.035290, 838063.221300, 859416.678362, 881139.576270, 903232.151726,
+ 925693.940877, 948524.600346, 971723.442348, 995289.632784, 1019222.059448, 1043519.402588, 1068180.163485, 1093202.854610,
+ 1118585.515820, 1144326.031845, 1170421.828997, 1196870.617641, 1223669.453959, 1250815.679160, 1278306.298944, 1306137.726017,
+ 1334306.782451, 1362809.672011, 1391643.048887, 1420802.665486, 1450284.419390, 1480084.556504, 1510198.362531, 1540621.709038,
+ 1571349.988582, 1602377.885766, 1633700.853485, 1665314.443974, 1697213.607484, 1729393.242718, 1761847.799154, 1794572.464804,
+ 1827562.002973, 1860811.213417, 1894314.858503, 1928067.079352, 1962062.774535, 1996296.400592, 2030762.354026, 2065455.981656,
+ 2100370.818573, 2135502.224529, 2170844.722018, 2206392.452100, 2242140.795895, 2278083.565225, 2314215.470175, 2350532.238358,
+ 2387028.028531, 2423697.739702, 2460535.617283, 2497537.908190, 2534699.627071, 2572014.508248, 2609478.240813, 2647086.878671,
+ 2684834.738503, 2722717.495987, 2760730.659130, 2798869.445379, 2837130.937688, 2875509.782410, 2914002.086936, 2952603.395700,
+ 2991310.971904, 3030119.629799, 3069026.891217, 3108027.543585, 3147118.485322, 3186296.724369, 3225557.893026, 3264899.405158,
+ 3304317.806643, 3343809.532357, 3383372.875494, 3423003.239113, 3462699.855290, 3502458.588120, 3542276.984489, 3582151.461666,
+ 3622080.455509, 3662062.087297, 3702093.774268, 3742173.351913, 3782298.542025, 3822466.599295, 3862676.059061, 3902925.301466,
+ 3943212.313188, 3983534.655139, 4023890.682932, 4064278.948819, 4104698.644146, 4145147.867936, 4185624.200116, 4226127.607823,
+ 4266655.471909, 4307207.261933, 4347781.577688, 4388376.075665, 4428991.084860, 4469625.512791, 4510278.322018, 4550949.135583,
+ 4591635.061142, 4632337.290243, 4673054.034167, 4713783.999697, 4754527.016825, 4795282.199016, 4836049.304645, 4876828.217063,
+ 4917617.432795, 4958416.033283, 4999223.332994, 5040040.200620, 5080865.836319, 5121698.006953, 5162537.663489, 5203383.861562,
+ 5244235.368756, 5285093.542054, 5325957.172545, 5366826.006832, 5407700.602295, 5448579.444589, 5489462.237514, 5530348.767163,
+ 5571240.286944, 5612135.490023, 5653033.590430, 5693935.090249, 5734840.012969, 5775747.413811, 5816658.159197, 5857571.727542,
+ 5898486.810748, 5939403.918361, 5980324.159335, 6021245.922584, 6062167.462275, 6103093.061724, 6144019.021028, 6184948.477708,
+ 6225878.111073, 6266809.926593, 6307741.380274, 6348674.587241, 6389609.653586, 6430546.024311, 6471482.661256, 6512419.658468,
+ 6553358.313587, 6594299.462793, 6635239.780670, 6676180.809115, 6717122.614963, 6758064.640997, 6799009.935719, 6839954.567726,
+ 6880900.950276, 6921847.965357, 6962794.580261, 7003742.234176, 7044691.155209, 7085639.223984, 7126587.984374, 7167538.597843,
+ 7208487.508674, 7249437.307369, 7290389.448350, 7331341.752887, 7372292.872774, 7413245.082344, 7454198.512431, 7495151.878502,
+ 7536105.815461, 7577059.755263, 7618013.175400, 7658967.579066, 7699922.475951, 7740878.015144, 7781833.511721, 7822789.986990,
+ 7863745.987999, 7904702.500036, 7945659.757785, 7986615.385020, 8027573.168319, 8068531.917672, 8109489.438412, 8150447.742963,
+ 8191403.350221, 8232362.807587, 8273321.255913, 8314281.043410, 8355240.292724, 8396199.214301, 8437156.471990, 8478114.862372,
+ 8519073.457296, 8560034.832965, 8600993.512814, 8641954.368755, 8682914.667061, 8723875.362809, 8764837.809961, 8805797.690078,
+ 8846758.789178, 8887720.671501, 8928680.126482, 8969640.467726, 9010601.160006, 9051561.769905, 9092522.487783, 9133484.594542,
+ 9174446.122191, 9215406.809237, 9256366.837569, 9297328.747584, 9338288.901510, 9379250.948160, 9420212.503729, 9461173.228802,
+ 9502133.928845, 9543094.489131, 9584055.513956, 9625015.618922, 9665975.737758, 9706937.403885, 9747896.035874, 9788856.165941,
+ 9829816.994627, 9870777.228642, 9911737.327683, 9952698.937964, 9993660.635143, 10034620.747904, 10075579.526207, 10116539.739394,
+ 10157498.625700, 10198458.468474, 10239418.486051, 10280376.431741, 10321335.494132, 10362295.312860, 10403254.402980, 10444214.519042,
+ 10485173.027410
+ },
+ // log K = 21
+ {
+ 1512674.959317, 1552434.197368, 1592928.155075, 1634158.593696, 1676126.959176, 1718834.017325, 1762279.726817, 1806464.610827,
+ 1851388.162884, 1897049.337397, 1943446.849678, 1990579.312415, 2038444.063365, 2087038.766553, 2136360.248426, 2186405.470599,
+ 2237170.640174, 2288651.015887, 2340842.752937, 2393739.986364, 2447337.958039, 2501630.515180, 2556611.361814, 2612274.289376,
+ 2668612.210872, 2725618.064986, 2783284.549430, 2841603.566294, 2900566.895724, 2960167.093438, 3020394.610851, 3081240.447614,
+ 3142696.078921, 3204751.105703, 3267397.808234, 3330624.834407, 3394422.872136, 3458781.966184, 3523690.805899, 3589140.856689,
+ 3655119.540256, 3721617.907853, 3788624.486645, 3856129.175227, 3924119.940026, 3992586.658818, 4061519.544440, 4130906.734075,
+ 4200737.500257, 4271000.909057, 4341685.842431, 4412781.306685, 4484277.199691, 4556162.287552, 4628426.495554, 4701059.642462,
+ 4774050.909933, 4847389.117100, 4921065.536469, 4995068.178578, 5069389.529141, 5144019.928509, 5218947.440841, 5294164.568050,
+ 5369660.364896, 5445425.915267, 5521454.727645, 5597734.642433, 5674257.411718, 5751014.463556, 5827999.822090, 5905203.759793,
+ 5982617.522812, 6060234.561845, 6138048.162353, 6216049.451496, 6294232.713331, 6372588.108087, 6451109.903764, 6529792.611180,
+ 6608629.912527, 6687615.097432, 6766740.086064, 6846000.116450, 6925391.839995, 7004908.093083, 7084544.127544, 7164293.822313,
+ 7244153.392582, 7324115.814384, 7404179.031163, 7484336.380737, 7564586.128001, 7644922.247694, 7725340.819662, 7805840.541693,
+ 7886412.416887, 7967054.301986, 8047764.087248, 8128539.447460, 8209376.477465, 8290273.766928, 8371225.884580, 8452230.183873,
+ 8533283.959811, 8614388.863382, 8695539.350564, 8776730.375553, 8857961.046732, 8939230.757845, 9020535.369667, 9101877.322129,
+ 9183251.420601, 9264654.743697, 9346087.109813, 9427548.372707, 9509034.441574, 9590547.753266, 9672081.682514, 9753639.600816,
+ 9835216.139507, 9916813.989591, 9998428.682824, 10080058.312759, 10161706.673718, 10243372.018016, 10325050.507034, 10406744.529470,
+ 10488447.296300, 10570163.109725, 10651893.512546, 10733632.712879, 10815382.384056, 10897140.573621, 10978907.646650, 11060681.383125,
+ 11142464.138730, 11224254.549571, 11306053.275368, 11387858.162207, 11469666.329635, 11551480.634292, 11633302.025569, 11715128.710512,
+ 11796961.403328, 11878795.670820, 11960634.652330, 12042477.348124, 12124323.950451, 12206172.128053, 12288028.811447, 12369886.306570,
+ 12451744.123236, 12533607.183832, 12615473.707341, 12697338.733240, 12779207.987968, 12861080.507718, 12942957.260823, 13024834.007606,
+ 13106715.252745, 13188597.806270, 13270478.933580, 13352362.482281, 13434247.263704, 13516133.477076, 13598020.834213, 13679909.264273,
+ 13761799.592878, 13843689.299819, 13925583.582108, 14007479.173806, 14089371.780210, 14171268.621526, 14253167.561983, 14335070.527750,
+ 14416971.620797, 14498873.784867, 14580778.789301, 14662681.662290, 14744586.311912, 14826493.286700, 14908400.702409, 14990305.605573,
+ 15072214.379125, 15154123.120545, 15236031.901296, 15317939.866920, 15399849.993076, 15481761.939120, 15563672.325616, 15645585.312945,
+ 15727497.406318, 15809411.780271, 15891325.875461, 15973239.145687, 16055155.936424, 16137069.783844, 16218984.663574, 16300899.732679,
+ 16382815.849682, 16464730.053474, 16546645.675799, 16628563.444017, 16710481.533611, 16792400.824636, 16874319.061886, 16956235.859045,
+ 17038151.394583, 17120072.629220, 17201992.200096, 17283914.365655, 17365836.161356, 17447756.880495, 17529681.545942, 17611601.675003,
+ 17693522.079185, 17775441.939609, 17857361.777214, 17939284.144858, 18021205.445833, 18103125.675947, 18185051.132189, 18266974.391463,
+ 18348894.689783, 18430817.526979, 18512736.992850, 18594657.001222, 18676577.636496, 18758497.698999, 18840421.426602, 18922345.360722,
+ 19004266.906157, 19086186.184618, 19168105.320223, 19250029.336282, 19331950.350200, 19413874.580826, 19495794.431194, 19577715.745558,
+ 19659638.264924, 19741557.363469, 19823478.431258, 19905400.862401, 19987318.682409, 20069238.585823, 20151157.549449, 20233076.181910,
+ 20314994.521260, 20396915.069140, 20478836.108717, 20560753.915833, 20642672.570035, 20724588.867440, 20806507.038063, 20888426.370670,
+ 20970344.006053
+ }
+ };
+ //CHECKSTYLE.ON: LineLength
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Conversions.java b/src/main/java/org/apache/datasketches/hll2/Conversions.java
new file mode 100644
index 000000000..a81a4c516
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Conversions.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.AUX_TOKEN;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+
+/**
+ * Converters for one TgtHllType to another. The source can be heap or direct, but the result is
+ * always on heap. These conversions only apply to sketches in HLL (dense) mode.
+ *
+ * @author Lee Rhodes
+ */
+class Conversions {
+
+ static final Hll4Array convertToHll4(final AbstractHllArray srcAbsHllArr) {
+ final int lgConfigK = srcAbsHllArr.getLgConfigK();
+ final Hll4Array hll4Array = new Hll4Array(lgConfigK);
+ hll4Array.putOutOfOrder(srcAbsHllArr.isOutOfOrder());
+
+ //1st pass: compute starting curMin and numAtCurMin:
+ final int pair = curMinAndNum(srcAbsHllArr);
+ final int curMin = HllUtil.getPairValue(pair);
+ final int numAtCurMin = HllUtil.getPairLow26(pair);
+
+ //2nd pass: Must know curMin to create AuxHashMap.
+ //Populate KxQ registers, build AuxHashMap if needed
+ final PairIterator itr = srcAbsHllArr.iterator();
+ AuxHashMap auxHashMap = hll4Array.getAuxHashMap(); //may be null
+ while (itr.nextValid()) {
+ final int slotNo = itr.getIndex();
+ final int actualValue = itr.getValue();
+ AbstractHllArray.hipAndKxQIncrementalUpdate(hll4Array, 0, actualValue);
+ if (actualValue >= (curMin + 15)) {
+ hll4Array.putNibble(slotNo, AUX_TOKEN);
+ if (auxHashMap == null) {
+ auxHashMap = new HeapAuxHashMap(LG_AUX_ARR_INTS[lgConfigK], lgConfigK);
+ hll4Array.putAuxHashMap(auxHashMap, false);
+ }
+ auxHashMap.mustAdd(slotNo, actualValue);
+ } else {
+ hll4Array.putNibble(slotNo, actualValue - curMin);
+ }
+ }
+
+ hll4Array.putCurMin(curMin);
+ hll4Array.putNumAtCurMin(numAtCurMin);
+ hll4Array.putHipAccum(srcAbsHllArr.getHipAccum()); //intentional overwrite
+ hll4Array.putRebuildCurMinNumKxQFlag(false);
+ return hll4Array;
+ }
+
+ /**
+ * This returns curMin and numAtCurMin as a pair and will be correct independent of the TgtHllType
+ * of the input AbstractHllArray.
+ *
+ * <p>In general, it is always true that for HLL_6 and HLL_8, curMin is always 0, and numAtCurMin
+ * is the number of zero slots. For these two types there is no need to track curMin nor to track
+ * numAtCurMin once all the slots are filled.
+ *
+ * @param absHllArr an instance of AbstractHllArray
+ * @return pair values representing numAtCurMin and curMin
+ */
+ static final int curMinAndNum(final AbstractHllArray absHllArr) {
+ int curMin = 64;
+ int numAtCurMin = 0;
+ final PairIterator itr = absHllArr.iterator();
+ while (itr.nextAll()) {
+ final int v = itr.getValue();
+ if (v > curMin) { continue; }
+ if (v < curMin) {
+ curMin = v;
+ numAtCurMin = 1;
+ } else {
+ numAtCurMin++;
+ }
+ }
+ return HllUtil.pair(numAtCurMin, curMin);
+ }
+
+ static final Hll6Array convertToHll6(final AbstractHllArray srcAbsHllArr) {
+ final int lgConfigK = srcAbsHllArr.lgConfigK;
+ final Hll6Array hll6Array = new Hll6Array(lgConfigK);
+ hll6Array.putOutOfOrder(srcAbsHllArr.isOutOfOrder());
+ int numZeros = 1 << lgConfigK;
+ final PairIterator itr = srcAbsHllArr.iterator();
+ while (itr.nextAll()) {
+ if (itr.getValue() != EMPTY) {
+ numZeros--;
+ hll6Array.couponUpdate(itr.getPair()); //couponUpdate creates KxQ registers
+ }
+ }
+ hll6Array.putNumAtCurMin(numZeros);
+ hll6Array.putHipAccum(srcAbsHllArr.getHipAccum()); //intentional overwrite
+ hll6Array.putRebuildCurMinNumKxQFlag(false);
+ return hll6Array;
+ }
+
+ static final Hll8Array convertToHll8(final AbstractHllArray srcAbsHllArr) {
+ final int lgConfigK = srcAbsHllArr.lgConfigK;
+ final Hll8Array hll8Array = new Hll8Array(lgConfigK);
+ hll8Array.putOutOfOrder(srcAbsHllArr.isOutOfOrder());
+ int numZeros = 1 << lgConfigK;
+ final PairIterator itr = srcAbsHllArr.iterator();
+ while (itr.nextAll()) {
+ if (itr.getValue() != EMPTY) {
+ numZeros--;
+ hll8Array.couponUpdate(itr.getPair()); //creates KxQ registers
+ }
+ }
+ hll8Array.putNumAtCurMin(numZeros);
+ hll8Array.putHipAccum(srcAbsHllArr.getHipAccum()); //intentional overwrite
+ hll8Array.putRebuildCurMinNumKxQFlag(false);
+ return hll8Array;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CouponHashSet.java b/src/main/java/org/apache/datasketches/hll2/CouponHashSet.java
new file mode 100644
index 000000000..8b60964df
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CouponHashSet.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_SET_SIZE;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_DENOM;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_NUMER;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_INT_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_INT_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.computeLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCurMode;
+import static org.apache.datasketches.hll2.PreambleUtil.extractHashSetCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractInt;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.extractTgtHllType;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class CouponHashSet extends CouponList {
+
+ /**
+ * Constructs this sketch with the intent of loading it with data
+ * @param lgConfigK the configured Lg K
+ * @param tgtHllType the new target Hll type
+ */
+ CouponHashSet(final int lgConfigK, final TgtHllType tgtHllType) {
+ super(lgConfigK, tgtHllType, CurMode.SET);
+ assert lgConfigK > 7;
+ }
+
+ /**
+ * Copy constructor
+ * @param that another CouponHashSet
+ */
+ CouponHashSet(final CouponHashSet that) {
+ super(that);
+ }
+
+ /**
+ * Copy As constructor.
+ * @param that another CouponHashSet
+ * @param tgtHllType the new target Hll type
+ */
+ CouponHashSet(final CouponHashSet that, final TgtHllType tgtHllType) {
+ super(that, tgtHllType);
+ }
+
+ //will also accept List, but results in a Set
+ static CouponHashSet heapifySet(final MemorySegment seg) {
+ final int lgConfigK = extractLgK(seg);
+ final TgtHllType tgtHllType = extractTgtHllType(seg);
+
+ final CurMode curMode = extractCurMode(seg);
+ final int segArrStart = (curMode == CurMode.LIST) ? LIST_INT_ARR_START : HASH_SET_INT_ARR_START;
+ final CouponHashSet set = new CouponHashSet(lgConfigK, tgtHllType);
+ final boolean segIsCompact = extractCompactFlag(seg);
+ final int couponCount = extractHashSetCount(seg);
+ int lgCouponArrInts = extractLgArr(seg);
+ if (lgCouponArrInts < LG_INIT_SET_SIZE) {
+ lgCouponArrInts = computeLgArr(seg, couponCount, lgConfigK);
+ }
+ if (segIsCompact) {
+ for (int i = 0; i < couponCount; i++) {
+ set.couponUpdate(extractInt(seg, segArrStart + (i << 2)));
+ }
+ } else { //updatable
+ set.couponCount = couponCount;
+ set.lgCouponArrInts = lgCouponArrInts;
+ final int couponArrInts = 1 << lgCouponArrInts;
+ set.couponIntArr = new int[couponArrInts];
+ MemorySegment.copy(seg, JAVA_INT_UNALIGNED, HASH_SET_INT_ARR_START, set.couponIntArr, 0, couponArrInts);
+ }
+ return set;
+ }
+
+ @Override
+ CouponHashSet copy() {
+ return new CouponHashSet(this);
+ }
+
+ @Override
+ CouponHashSet copyAs(final TgtHllType tgtHllType) {
+ return new CouponHashSet(this, tgtHllType);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ final int index = find(couponIntArr, lgCouponArrInts, coupon);
+ if (index >= 0) {
+ return this; //found duplicate, ignore
+ }
+ couponIntArr[~index] = coupon; //found empty
+ couponCount++;
+ if (checkGrowOrPromote()) {
+ return promoteHeapListOrSetToHll(this);
+ }
+ return this;
+ }
+
+ @Override
+ int getSegDataStart() {
+ return HASH_SET_INT_ARR_START;
+ }
+
+ @Override
+ int getPreInts() {
+ return HASH_SET_PREINTS;
+ }
+
+ private boolean checkGrowOrPromote() {
+ if ((RESIZE_DENOM * couponCount) > (RESIZE_NUMER * (1 << lgCouponArrInts))) {
+ if (lgCouponArrInts == (lgConfigK - 3)) { //at max size
+ return true; // promote to HLL
+ }
+ couponIntArr = growHashSet(couponIntArr, ++lgCouponArrInts);
+ }
+ return false;
+ }
+
+ private static int[] growHashSet(final int[] coupIntArr, final int tgtLgCoupArrSize) {
+ final int[] tgtCouponIntArr = new int[1 << tgtLgCoupArrSize]; //create tgt
+ final int len = coupIntArr.length;
+ for (int i = 0; i < len; i++) { //scan input arr for non-zero values
+ final int fetched = coupIntArr[i];
+ if (fetched != EMPTY) {
+ final int idx = find(tgtCouponIntArr, tgtLgCoupArrSize, fetched); //find empty in tgt
+ if (idx < 0) { //found EMPTY
+ tgtCouponIntArr[~idx] = fetched; //insert
+ continue;
+ }
+ throw new SketchesStateException("Error: found duplicate.");
+ }
+ }
+ return tgtCouponIntArr;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CouponList.java b/src/main/java/org/apache/datasketches/hll2/CouponList.java
new file mode 100644
index 000000000..78337414c
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CouponList.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_LIST_SIZE;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_SET_SIZE;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_INT_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.extractListCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractTgtHllType;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+class CouponList extends AbstractCoupons {
+ int lgCouponArrInts;
+ int couponCount;
+ int[] couponIntArr;
+
+ private static int checkLgConfigK(final CurMode curMode, final int lgConfigK) {
+ if (curMode == CurMode.SET) { assert lgConfigK > 7; }
+ return lgConfigK;
+ }
+
+ /**
+ * New instance constructor for LIST or SET.
+ * @param lgConfigK the configured Lg K
+ * @param tgtHllType the configured HLL target
+ * @param curMode LIST or SET
+ */
+ CouponList(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode) {
+ super(checkLgConfigK(curMode, lgConfigK), tgtHllType, curMode);
+ if (curMode == CurMode.LIST) {
+ lgCouponArrInts = LG_INIT_LIST_SIZE;
+ } else { //SET
+ lgCouponArrInts = LG_INIT_SET_SIZE;
+ }
+ couponIntArr = new int[1 << lgCouponArrInts];
+ couponCount = 0;
+ }
+
+ /**
+ * Copy Constructor
+ * @param that another CouponArray
+ */
+ CouponList(final CouponList that) {
+ super(that.lgConfigK, that.tgtHllType, that.curMode);
+ lgCouponArrInts = that.lgCouponArrInts;
+ couponCount = that.couponCount;
+ couponIntArr = that.couponIntArr.clone();
+ }
+
+ /**
+ * Copy As constructor.
+ * @param that another CouponList
+ * @param tgtHllType the new target Hll type
+ */ //also used by CouponHashSet
+ CouponList(final CouponList that, final TgtHllType tgtHllType) {
+ super(that.lgConfigK, tgtHllType, that.curMode);
+ lgCouponArrInts = that.lgCouponArrInts;
+ couponCount = that.couponCount;
+ couponIntArr = that.couponIntArr.clone();
+ }
+
+ static final CouponList heapifyList(final MemorySegment seg) {
+ final int lgConfigK = extractLgK(seg);
+ final TgtHllType tgtHllType = extractTgtHllType(seg);
+
+ final CouponList list = new CouponList(lgConfigK, tgtHllType, CurMode.LIST);
+ final int couponCount = extractListCount(seg);
+ MemorySegment.copy(seg, JAVA_INT_UNALIGNED, LIST_INT_ARR_START, list.couponIntArr, 0, couponCount);
+ list.couponCount = couponCount;
+ return list;
+ }
+
+ @Override
+ CouponList copy() {
+ return new CouponList(this);
+ }
+
+ @Override
+ CouponList copyAs(final TgtHllType tgtHllType) {
+ return new CouponList(this, tgtHllType);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ final int len = 1 << lgCouponArrInts;
+ for (int i = 0; i < len; i++) { //search for empty slot
+ final int couponAtIdx = couponIntArr[i];
+ if (couponAtIdx == EMPTY) {
+ couponIntArr[i] = coupon; //update
+ couponCount++;
+ if (couponCount >= len) { //array full
+ if (lgConfigK < 8) {
+ return promoteHeapListOrSetToHll(this); //oooFlag = false
+ }
+ return promoteHeapListToSet(this); //oooFlag = true
+ }
+ return this;
+ }
+ //cell not empty
+ if (couponAtIdx == coupon) {
+ return this; //duplicate
+ }
+ //cell not empty & not a duplicate, continue
+ } //end for
+ throw new SketchesStateException("Array invalid: no empties & no duplicates");
+ }
+
+ @Override
+ int getCompactSerializationBytes() {
+ return getSegDataStart() + (couponCount << 2);
+ }
+
+ @Override
+ int getCouponCount() {
+ return couponCount;
+ }
+
+ @Override
+ int[] getCouponIntArr() {
+ return couponIntArr;
+ }
+
+ @Override
+ int getLgCouponArrInts() {
+ return lgCouponArrInts;
+ }
+
+ @Override
+ int getSegDataStart() {
+ return LIST_INT_ARR_START;
+ }
+
+ @Override
+ MemorySegment getMemorySegment() {
+ return null;
+ }
+
+ @Override
+ int getPreInts() {
+ return LIST_PREINTS;
+ }
+
+ @Override
+ boolean isCompact() {
+ return false;
+ }
+
+ @Override
+ boolean hasMemorySegment() {
+ return false;
+ }
+
+ @Override
+ boolean isOffHeap() {
+ return false;
+ }
+
+ @Override
+ boolean isSameResource(final MemorySegment seg) {
+ return false;
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new IntArrayPairIterator(couponIntArr, lgConfigK);
+ }
+
+ @Override
+ void mergeTo(final HllSketch that) {
+ final int arrLen = couponIntArr.length;
+ for (int i = 0; i < arrLen; i++) {
+ final int pair = couponIntArr[i];
+ if (pair == 0) { continue; }
+ that.couponUpdate(pair);
+ }
+ }
+
+ @Override
+ CouponList reset() {
+ return new CouponList(lgConfigK, tgtHllType, CurMode.LIST);
+ }
+
+ static final HllSketchImpl promoteHeapListToSet(final CouponList list) {
+ final int couponCount = list.couponCount;
+ final int[] arr = list.couponIntArr;
+ final CouponHashSet chSet = new CouponHashSet(list.lgConfigK, list.tgtHllType);
+ for (int i = 0; i < couponCount; i++) {
+ chSet.couponUpdate(arr[i]);
+ }
+ return chSet;
+ }
+
+ //Promotional move of coupons to an HllSketch from either List or Set.
+ //called by CouponHashSet.couponUpdate()
+ //called by CouponList.couponUpdate()
+ static final HllSketchImpl promoteHeapListOrSetToHll(final CouponList src) {
+ final HllArray tgtHllArr = HllArray.newHeapHll(src.lgConfigK, src.tgtHllType);
+ final PairIterator srcItr = src.iterator();
+ tgtHllArr.putKxQ0(1 << src.lgConfigK);
+ while (srcItr.nextValid()) {
+ tgtHllArr.couponUpdate(srcItr.getPair());
+ }
+ tgtHllArr.putHipAccum(src.getEstimate());
+
+ tgtHllArr.putOutOfOrder(false);
+ return tgtHllArr;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CouponMapping.java b/src/main/java/org/apache/datasketches/hll2/CouponMapping.java
new file mode 100644
index 000000000..5ea38a30c
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CouponMapping.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ *
+ */
+final class CouponMapping {
+
+ //Computed for Coupon lgK = 26 ONLY. Designed for the cubic interpolator function.
+ static final double[] xArr = new double[] {
+ 0.0, 1.0, 20.0, 400.0,
+ 8000.0, 160000.0, 300000.0, 600000.0,
+ 900000.0, 1200000.0, 1500000.0, 1800000.0,
+ 2100000.0, 2400000.0, 2700000.0, 3000000.0,
+ 3300000.0, 3600000.0, 3900000.0, 4200000.0,
+ 4500000.0, 4800000.0, 5100000.0, 5400000.0,
+ 5700000.0, 6000000.0, 6300000.0, 6600000.0,
+ 6900000.0, 7200000.0, 7500000.0, 7800000.0,
+ 8100000.0, 8400000.0, 8700000.0, 9000000.0,
+ 9300000.0, 9600000.0, 9900000.0, 10200000.0
+ };
+
+ // CHECKSTYLE:OFF LineLength
+ //Computed for Coupon lgK = 26 ONLY. Designed for the cubic interpolator function.
+ static final double[] yArr = new double[] {
+ 0.0000000000000000, 1.0000000000000000, 20.0000009437402611, 400.0003963713384110,
+ 8000.1589294602090376, 160063.6067763759638183, 300223.7071597663452849, 600895.5933856170158833,
+ 902016.8065120954997838, 1203588.4983199508860707, 1505611.8245524743106216, 1808087.9449319066479802,
+ 2111018.0231759352609515, 2414403.2270142501220107, 2718244.7282051891088486, 3022543.7025524540804327,
+ 3327301.3299219091422856, 3632518.7942584538832307, 3938197.2836029687896371, 4244337.9901093561202288,
+ 4550942.1100616492331028, 4858010.8438911894336343, 5165545.3961938973516226, 5473546.9757476449012756,
+ 5782016.7955296505242586, 6090956.0727340159937739, 6400366.0287892958149314, 6710247.8893762007355690,
+ 7020602.8844453142955899, 7331432.2482349723577499, 7642737.2192891482263803, 7954519.0404754765331745,
+ 8266778.9590033423155546, 8579518.2264420464634895, 8892738.0987390466034412, 9206439.8362383283674717,
+ 9520624.7036988288164139, 9835293.9703129194676876, 10150448.9097250290215015, 10466090.8000503256917000
+ };
+ // CHECKSTYLE:ON LineLength
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CubicInterpolation.java b/src/main/java/org/apache/datasketches/hll2/CubicInterpolation.java
new file mode 100644
index 000000000..8797201b7
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CubicInterpolation.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class CubicInterpolation {
+
+ /**
+ * Cubic interpolation using interpolation X and Y tables.
+ *
+ * @param xArr xArr
+ * @param yArr yArr
+ * @param x x
+ * @return cubic interpolation
+ */
+ //Used by AbstractCoupons
+ //In C: again-two-registers cubic_interpolate_using_table L1377
+ static double usingXAndYTables(final double[] xArr, final double[] yArr,
+ final double x) {
+ assert (xArr.length >= 4) && (xArr.length == yArr.length);
+ if ((x < xArr[0]) || (x > xArr[xArr.length - 1])) {
+ throw new SketchesArgumentException("X value out of range: " + x);
+ }
+ if (x == xArr[xArr.length - 1]) {
+ return yArr[yArr.length - 1]; // corner case
+ }
+ final int offset = findStraddle(xArr, x); //uses recursion
+ assert (offset >= 0) && (offset <= (xArr.length - 2));
+ if (offset == 0) {
+ return interpolateUsingXAndYTables(xArr, yArr, offset, x); // corner case
+ }
+ if (offset == (xArr.length - 2)) {
+ return interpolateUsingXAndYTables(xArr, yArr, offset - 2, x); // corner case
+ }
+ return interpolateUsingXAndYTables(xArr, yArr, offset - 1, x);
+ }
+
+ // In C: again-two-registers cubic_interpolate_aux L1368
+ private static double interpolateUsingXAndYTables(final double[] xArr, final double[] yArr,
+ final int offset, final double x) {
+ return cubicInterpolate(xArr[offset], yArr[offset], xArr[offset + 1], yArr[offset + 1],
+ xArr[offset + 2], yArr[offset + 2], xArr[offset + 3], yArr[offset + 3], x);
+ }
+
+ //Interpolate using X table and Y stride
+
+ /**
+ * Cubic interpolation using interpolation X table and Y stride.
+ *
+ * @param xArr The x array
+ * @param yStride The y stride
+ * @param x The value x
+ * @return cubic interpolation
+ */
+ //In C: again-two-registers cubic_interpolate_with_x_arr_and_y_stride L1411
+ // Used by HllEstimators
+ static double usingXArrAndYStride(
+ final double[] xArr, final double yStride, final double x) {
+ final int xArrLen = xArr.length;
+ final int xArrLenM1 = xArrLen - 1;
+
+ final int offset;
+ assert ((xArrLen >= 4) && (x >= xArr[0]) && (x <= xArr[xArrLenM1]));
+
+ if (x == xArr[xArrLenM1]) { /* corner case */
+ return (yStride * (xArrLenM1));
+ }
+
+ offset = findStraddle(xArr, x); //uses recursion
+ final int xArrLenM2 = xArrLen - 2;
+ assert ((offset >= 0) && (offset <= (xArrLenM2)));
+
+ if (offset == 0) { /* corner case */
+ return (interpolateUsingXArrAndYStride(xArr, yStride, (offset - 0), x));
+ }
+ else if (offset == xArrLenM2) { /* corner case */
+ return (interpolateUsingXArrAndYStride(xArr, yStride, (offset - 2), x));
+ }
+ /* main case */
+ return (interpolateUsingXArrAndYStride(xArr, yStride, (offset - 1), x));
+ }
+
+ //In C: again-two-registers cubic_interpolate_with_x_arr_and_y_stride_aux L1402
+ private static double interpolateUsingXArrAndYStride(final double[] xArr, final double yStride,
+ final int offset, final double x) {
+ return cubicInterpolate(
+ xArr[offset + 0], yStride * (offset + 0),
+ xArr[offset + 1], yStride * (offset + 1),
+ xArr[offset + 2], yStride * (offset + 2),
+ xArr[offset + 3], yStride * (offset + 3),
+ x);
+ }
+
+ //Cubic Interpolation used by both methods
+
+ // Interpolate using the cubic curve that passes through the four given points, using the
+ // Lagrange interpolation formula.
+ // In C: again-two-registers cubic_interpolate_aux_aux L1346
+ private static double cubicInterpolate(final double x0, final double y0, final double x1,
+ final double y1, final double x2, final double y2, final double x3, final double y3,
+ final double x) {
+ final double l0Numer = (x - x1) * (x - x2) * (x - x3);
+ final double l1Numer = (x - x0) * (x - x2) * (x - x3);
+ final double l2Numer = (x - x0) * (x - x1) * (x - x3);
+ final double l3Numer = (x - x0) * (x - x1) * (x - x2);
+
+ final double l0Denom = (x0 - x1) * (x0 - x2) * (x0 - x3);
+ final double l1Denom = (x1 - x0) * (x1 - x2) * (x1 - x3);
+ final double l2Denom = (x2 - x0) * (x2 - x1) * (x2 - x3);
+ final double l3Denom = (x3 - x0) * (x3 - x1) * (x3 - x2);
+
+ final double term0 = (y0 * l0Numer) / l0Denom;
+ final double term1 = (y1 * l1Numer) / l1Denom;
+ final double term2 = (y2 * l2Numer) / l2Denom;
+ final double term3 = (y3 * l3Numer) / l3Denom;
+
+ return term0 + term1 + term2 + term3;
+ }
+
+ //In C: again-two-registers.c find_straddle L1335
+ private static int findStraddle(final double[] xArr, final double x) {
+ assert ((xArr.length >= 2) && (x >= xArr[0]) && (x <= xArr[xArr.length - 1]));
+ return (recursiveFindStraddle(xArr, 0, xArr.length - 1, x));
+ }
+
+ //In C: again-two-registers.c find_straddle_aux L1322
+ private static int recursiveFindStraddle(final double[] xArr, final int left, final int right,
+ final double x) {
+ final int middle;
+ assert (left < right);
+ assert ((xArr[left] <= x) && (x < xArr[right])); /* the invariant */
+ if ((left + 1) == right) {
+ return (left);
+ }
+ middle = left + ((right - left) / 2);
+ if (xArr[middle] <= x) {
+ return (recursiveFindStraddle(xArr, middle, right, x));
+ } else {
+ return (recursiveFindStraddle(xArr, left, middle, x));
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/CurMode.java b/src/main/java/org/apache/datasketches/hll2/CurMode.java
new file mode 100644
index 000000000..cab1d2fd8
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/CurMode.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * Represents the three fundamental modes of the HLL Sketch.
+ *
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+enum CurMode { LIST, SET, HLL; //do not change the order.
+
+ public static final CurMode values[] = values();
+
+ /**
+ * Returns the CurMode given its ordinal
+ * @param ordinal the order of appearance in the enum definition.
+ * @return the CurMode given its ordinal
+ */
+ public static CurMode fromOrdinal(final int ordinal) {
+ return values[ordinal];
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll2/DirectAuxHashMap.java
new file mode 100644
index 000000000..2f44b1426
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectAuxHashMap.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.common.Util.clear;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_DENOM;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_NUMER;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.extractAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.insertAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+final class DirectAuxHashMap implements AuxHashMap {
+ private final DirectHllArray host; //hosts the MemorySegment and read-only MemorySegment
+ private final boolean readOnly;
+
+ DirectAuxHashMap(final DirectHllArray host, final boolean initialize) {
+ this.host = host;
+ readOnly = (host.wseg == null);
+ final int initLgArrInts = HllUtil.LG_AUX_ARR_INTS[host.lgConfigK];
+
+ if (initialize) { //must be writable
+ if (readOnly) { noWriteAccess(); }
+ insertLgArr(host.wseg, initLgArrInts);
+ clear(host.wseg, host.auxStart, 4 << initLgArrInts);
+ } else if (extractLgArr(host.seg) < initLgArrInts) {
+ if (readOnly) {
+ throw new SketchesArgumentException(
+ "Possible MemorySegment image corruption, incorrect LgArr field in preamble.");
+ }
+ //insert the correct LgArr value
+ final int lgArr =
+ PreambleUtil.computeLgArr(host.wseg, host.auxHashMap.getAuxCount(), host.lgConfigK);
+ insertLgArr(host.wseg, lgArr);
+ }
+ }
+
+ @Override
+ public DirectAuxHashMap copy() { //a no-op
+ return null;
+ }
+
+ @Override
+ public int getAuxCount() {
+ return extractAuxCount(host.seg);
+ }
+
+ @Override
+ public int[] getAuxIntArr() {
+ return null;
+ }
+
+ @Override
+ public int getCompactSizeBytes() {
+ return getAuxCount() << 2;
+ }
+
+ @Override
+ public PairIterator getIterator() {
+ return new IntMemorySegmentPairIterator(
+ host.seg, host.auxStart, 1 << getLgAuxArrInts(), host.lgConfigK);
+ }
+
+ @Override
+ public int getLgAuxArrInts() {
+ return extractLgArr(host.seg);
+ }
+
+ @Override
+ public int getUpdatableSizeBytes() {
+ return 4 << getLgAuxArrInts();
+ }
+
+ @Override
+ public boolean hasMemorySegment() {
+ return host.hasMemorySegment();
+ }
+
+ @Override
+ public boolean isOffHeap() {
+ return host.isOffHeap();
+ }
+
+ @Override
+ public boolean isSameResource(final MemorySegment seg) {
+ return host.isSameResource(seg);
+ }
+
+ @Override
+ public void mustAdd(final int slotNo, final int value) {
+ if (readOnly) { noWriteAccess(); }
+ final int index = find(host, slotNo);
+ final int pair = HllUtil.pair(slotNo, value);
+ if (index >= 0) {
+ final String pairStr = HllUtil.pairString(pair);
+ throw new SketchesStateException("Found a slotNo that should not be there: " + pairStr);
+ }
+ //Found empty entry
+ host.wseg.set(JAVA_INT_UNALIGNED, host.auxStart + (~index << 2), pair);
+ int auxCount = extractAuxCount(host.seg);
+ insertAuxCount(host.wseg, ++auxCount);
+ final int lgAuxArrInts = extractLgArr(host.seg);
+ if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * (1 << lgAuxArrInts))) {
+ grow(host, lgAuxArrInts);
+ }
+ }
+
+ @Override
+ public int mustFindValueFor(final int slotNo) {
+ final int index = find(host, slotNo);
+ if (index >= 0) {
+ final int pair = host.seg.get(JAVA_INT_UNALIGNED, host.auxStart + (index << 2));
+ return HllUtil.getPairValue(pair);
+ }
+ throw new SketchesStateException("SlotNo not found: " + slotNo);
+ }
+
+ @Override
+ public void mustReplace(final int slotNo, final int value) {
+ if (readOnly) { noWriteAccess(); }
+ final int index = find(host, slotNo);
+ if (index >= 0) {
+ host.wseg.set(JAVA_INT_UNALIGNED, host.auxStart + (index << 2), HllUtil.pair(slotNo, value));
+ return;
+ }
+ final String pairStr = HllUtil.pairString(HllUtil.pair(slotNo, value));
+ throw new SketchesStateException("Pair not found: " + pairStr);
+ }
+
+ //Searches the Aux arr hash table (embedded in MemorySegment) for an empty or a matching slotNo
+ // depending on the context.
+ //If entire entry is empty, returns one's complement of index = found empty.
+ //If entry contains given slotNo, returns its index = found slotNo.
+ //Continues searching.
+ //If the probe comes back to original index, throws an exception.
+ private static int find(final DirectHllArray host, final int slotNo) {
+ final int lgAuxArrInts = extractLgArr(host.seg);
+ assert lgAuxArrInts < host.lgConfigK : lgAuxArrInts;
+ final int auxInts = 1 << lgAuxArrInts;
+ final int auxArrMask = auxInts - 1;
+ final int configKmask = (1 << host.lgConfigK) - 1;
+ int probe = slotNo & auxArrMask;
+ final int loopIndex = probe;
+ do {
+ final int arrVal = host.seg.get(JAVA_INT_UNALIGNED, host.auxStart + (probe << 2));
+ if (arrVal == EMPTY) {
+ return ~probe; //empty
+ }
+ else if (slotNo == (arrVal & configKmask)) { //found given slotNo
+ return probe; //return aux array index
+ }
+ final int stride = (slotNo >>> lgAuxArrInts) | 1;
+ probe = (probe + stride) & auxArrMask;
+ } while (probe != loopIndex);
+ throw new SketchesArgumentException("Key not found and no empty slots!");
+ }
+
+ private static void grow(final DirectHllArray host, final int oldLgAuxArrInts) {
+ if (host.wseg == null) { noWriteAccess(); }
+ final int oldAuxArrInts = 1 << oldLgAuxArrInts;
+ final int[] oldIntArray = new int[oldAuxArrInts]; //buffer old aux data
+ MemorySegment.copy(host.wseg, JAVA_INT_UNALIGNED, host.auxStart, oldIntArray, 0, oldAuxArrInts);
+
+ insertLgArr(host.wseg, oldLgAuxArrInts + 1); //update LgArr field
+
+ final long newAuxBytes = oldAuxArrInts << 3;
+ final long requestBytes = host.auxStart + newAuxBytes;
+ final long oldCapBytes = host.wseg.byteSize();
+
+ if (requestBytes > oldCapBytes) {
+ final MemorySegment newWseg = MemorySegment.ofArray(new byte[(int)requestBytes]);
+ MemorySegment.copy(host.wseg, 0, newWseg, 0, host.auxStart);
+
+ clear(newWseg, host.auxStart, newAuxBytes); //clear space for new aux data
+ host.updateMemorySegment(newWseg);
+ }
+ //rehash into larger aux array
+ final int configKmask = (1 << host.lgConfigK) - 1;
+
+ for (int i = 0; i < oldAuxArrInts; i++) {
+ final int fetched = oldIntArray[i];
+ if (fetched != EMPTY) {
+ //find empty in new array
+ final int index = find(host, fetched & configKmask);
+ host.wseg.set(JAVA_INT_UNALIGNED, host.auxStart + (~index << 2), fetched);
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectCouponHashSet.java b/src/main/java/org/apache/datasketches/hll2/DirectCouponHashSet.java
new file mode 100644
index 000000000..0b11f3b7b
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectCouponHashSet.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.common.Util.clear;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.KEY_MASK_26;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_DENOM;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_NUMER;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_INT_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.LG_K_BYTE;
+import static org.apache.datasketches.hll2.PreambleUtil.extractHashSetCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractInt;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.insertHashSetCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertInt;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesException;
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+final class DirectCouponHashSet extends DirectCouponList {
+
+ //Constructs this sketch with data.
+ DirectCouponHashSet(final int lgConfigK, final TgtHllType tgtHllType, final MemorySegment wseg) {
+ super(lgConfigK, tgtHllType, CurMode.SET, wseg);
+ assert wseg.get(JAVA_BYTE, LG_K_BYTE) > 7;
+ }
+
+ //Constructs this sketch with read-only data, may be compact.
+ DirectCouponHashSet(final int lgConfigK, final TgtHllType tgtHllType, final MemorySegment seg,
+ final boolean readOnly) {
+ super(lgConfigK, tgtHllType, CurMode.SET, seg, true);
+ assert seg.get(JAVA_BYTE, LG_K_BYTE) > 7;
+ }
+
+ @Override //returns on-heap Set
+ CouponHashSet copy() {
+ return CouponHashSet.heapifySet(seg);
+ }
+
+ @Override //returns on-heap Set
+ CouponHashSet copyAs(final TgtHllType tgtHllType) {
+ final CouponHashSet clist = CouponHashSet.heapifySet(seg);
+ return new CouponHashSet(clist, tgtHllType);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ if (wseg == null) { noWriteAccess(); }
+ //avoid array copy
+ final int index = find(seg, getLgCouponArrInts(), coupon);
+ if (index >= 0) {
+ return this; //found duplicate, ignore
+ }
+ insertInt(wseg, HASH_SET_INT_ARR_START + (~index << 2), coupon);
+ insertHashSetCount(wseg, getCouponCount() + 1);
+ final boolean promote = checkGrowOrPromote();
+ if (!promote) { return this; }
+ return promoteListOrSetToHll(this);
+ }
+
+ @Override
+ int getCouponCount() {
+ return extractHashSetCount(seg);
+ }
+
+ @Override
+ int getSegDataStart() {
+ return HASH_SET_INT_ARR_START;
+ }
+
+ @Override
+ int getPreInts() {
+ return HASH_SET_PREINTS;
+ }
+
+ private boolean checkGrowOrPromote() {
+ int lgCouponArrInts = getLgCouponArrInts();
+ if ((RESIZE_DENOM * getCouponCount()) > (RESIZE_NUMER * (1 << lgCouponArrInts))) {
+ if (lgCouponArrInts == (getLgConfigK() - 3)) {
+ return true; // promote
+ }
+ insertLgArr(wseg, ++lgCouponArrInts);
+ growHashSet(wseg, lgCouponArrInts);
+ }
+ return false;
+ }
+
+ //This could fail if the user has undersized the given MemorySegment
+ // and not used the public methods for sizing the MemorySegment. See exception.
+ private static void growHashSet(final MemorySegment wseg, final int tgtLgCouponArrSize) {
+ final int tgtArrSize = 1 << tgtLgCouponArrSize;
+ final int[] tgtCouponIntArr = new int[tgtArrSize];
+ final int oldLen = 1 << extractLgArr(wseg);
+ for (int i = 0; i < oldLen; i++) {
+ final int fetched = extractInt(wseg, HASH_SET_INT_ARR_START + (i << 2));
+ if (fetched != EMPTY) {
+ final int idx = find(tgtCouponIntArr, tgtLgCouponArrSize, fetched);
+ if (idx < 0) { //found EMPTY
+ tgtCouponIntArr[~idx] = fetched; //insert
+ continue;
+ }
+ throw new SketchesStateException("Error: found duplicate.");
+ }
+ }
+ clear(wseg, HASH_SET_INT_ARR_START, tgtArrSize << 2);
+ try { MemorySegment.copy(tgtCouponIntArr, 0, wseg, JAVA_INT_UNALIGNED, HASH_SET_INT_ARR_START, tgtArrSize); }
+ catch (final IndexOutOfBoundsException e) {
+ throw new SketchesException(
+ "The MemorySegment is undersized. Use the public methods for properly sizing MemorySegment.", e);
+ }
+ }
+
+ //Searches the Coupon hash table (embedded in MemorySegment) for an empty slot
+ // or a duplicate depending on the context.
+ //If entire entry is empty, returns one's complement of index = found empty.
+ //If entry equals given coupon, returns its index = found duplicate coupon
+ //Continues searching
+ //If the probe comes back to original index, throws an exception.
+ private static int find(final MemorySegment seg, final int lgArr,
+ final int coupon) {
+ final int arrMask = (1 << lgArr) - 1;
+ int probe = coupon & arrMask;
+ final int loopIndex = probe;
+ do {
+ final int couponAtIndex = extractInt(seg, HASH_SET_INT_ARR_START + (probe << 2));
+ if (couponAtIndex == EMPTY) { return ~probe; } //empty
+ else if (coupon == couponAtIndex) { return probe; } //duplicate
+ final int stride = ((coupon & KEY_MASK_26) >>> lgArr) | 1;
+ probe = (probe + stride) & arrMask;
+ } while (probe != loopIndex);
+ throw new SketchesArgumentException("Key not found and no empty slots!");
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectCouponList.java b/src/main/java/org/apache/datasketches/hll2/DirectCouponList.java
new file mode 100644
index 000000000..4e9c5136e
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectCouponList.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.common.Util.clear;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_LIST_SIZE;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_SET_SIZE;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.EMPTY_FLAG_MASK;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_INT_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.computeLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractInt;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.extractListCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCurMode;
+import static org.apache.datasketches.hll2.PreambleUtil.insertEmptyFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertFamilyId;
+import static org.apache.datasketches.hll2.PreambleUtil.insertFlags;
+import static org.apache.datasketches.hll2.PreambleUtil.insertInt;
+import static org.apache.datasketches.hll2.PreambleUtil.insertKxQ0;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.insertListCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertModes;
+import static org.apache.datasketches.hll2.PreambleUtil.insertNumAtCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertPreInts;
+import static org.apache.datasketches.hll2.PreambleUtil.insertSerVer;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.MemorySegmentStatus;
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+class DirectCouponList extends AbstractCoupons {
+ MemorySegment wseg;
+ MemorySegment seg;
+ final boolean compact;
+
+ private static int checkSegCompactFlag(final MemorySegment wseg, final int lgConfigK) {
+ assert !extractCompactFlag(wseg) : "Compact Flag must not be set.";
+ return lgConfigK;
+ }
+
+ //called from newInstance, writableWrap and DirectCouponHashSet, must not be compact
+ DirectCouponList(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode, final MemorySegment wseg) {
+ super(checkSegCompactFlag(wseg, lgConfigK), tgtHllType, curMode);
+ this.wseg = wseg;
+ seg = wseg;
+ compact = extractCompactFlag(wseg);
+ }
+
+ //called from HllSketch.wrap and from DirectCouponHashSet constructor, may or may not be compact
+ DirectCouponList(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode, final MemorySegment seg,
+ final boolean readOnly) {
+ super(lgConfigK, tgtHllType, curMode);
+ wseg = null;
+ this.seg = seg;
+ compact = extractCompactFlag(seg);
+ }
+
+ /**
+ * Standard factory for new DirectCouponList.
+ * This initializes the given MemorySegment.
+ * @param lgConfigK the configured Lg K
+ * @param tgtHllType the configured HLL target
+ * @param dstSeg the destination MemorySegment for the sketch.
+ * @return a new DirectCouponList
+ */
+ static DirectCouponList newInstance(final int lgConfigK, final TgtHllType tgtHllType,
+ final MemorySegment dstSeg) {
+ insertPreInts(dstSeg, LIST_PREINTS);
+ insertSerVer(dstSeg);
+ insertFamilyId(dstSeg);
+ insertLgK(dstSeg, lgConfigK);
+ insertLgArr(dstSeg, LG_INIT_LIST_SIZE);
+ insertFlags(dstSeg, EMPTY_FLAG_MASK); //empty and not compact
+ insertListCount(dstSeg, 0);
+ insertModes(dstSeg, tgtHllType, CurMode.LIST);
+ return new DirectCouponList(lgConfigK, tgtHllType, CurMode.LIST, dstSeg);
+ }
+
+ @Override //returns on-heap List
+ CouponList copy() {
+ return CouponList.heapifyList(seg);
+ }
+
+ @Override //returns on-heap List
+ CouponList copyAs(final TgtHllType tgtHllType) {
+ final CouponList clist = CouponList.heapifyList(seg);
+ return new CouponList(clist, tgtHllType);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ if (wseg == null) { noWriteAccess(); }
+ final int len = 1 << getLgCouponArrInts();
+ for (int i = 0; i < len; i++) { //search for empty slot and duplicates
+ final int couponAtIdx = extractInt(seg, LIST_INT_ARR_START + (i << 2));
+ if (couponAtIdx == EMPTY) {
+ insertInt(wseg, LIST_INT_ARR_START + (i << 2), coupon);
+ int couponCount = extractListCount(seg);
+ insertListCount(wseg, ++couponCount);
+ insertEmptyFlag(wseg, false); //only first time
+ if (couponCount >= len) { //array full
+ if (lgConfigK < 8) {
+ return promoteListOrSetToHll(this);//oooFlag = false
+ }
+ return promoteListToSet(this); //oooFlag = true
+ }
+ return this;
+ }
+ //cell not empty
+ if (couponAtIdx == coupon) { return this; } //return if duplicate
+ //cell not empty & not a duplicate, continue
+ } //end for
+ throw new SketchesStateException("Invalid State: no empties & no duplicates");
+ }
+
+ @Override
+ int getCompactSerializationBytes() {
+ return getSegDataStart() + (getCouponCount() << 2);
+ }
+
+ @Override //Overridden by DirectCouponHashSet
+ int getCouponCount() {
+ return extractListCount(seg);
+ }
+
+ @Override
+ int[] getCouponIntArr() { //here only to satisfy the abstract, should not be used
+ return null;
+ }
+
+ @Override
+ int getLgCouponArrInts() {
+ final int lgArr = extractLgArr(seg);
+ if (lgArr >= LG_INIT_LIST_SIZE) { return lgArr; }
+ //early versions of compact images did not use this lgArr field
+ final int coupons = getCouponCount();
+ return computeLgArr(seg, coupons, lgConfigK);
+ }
+
+ @Override
+ int getSegDataStart() {
+ return LIST_INT_ARR_START;
+ }
+
+ @Override
+ MemorySegment getMemorySegment() {
+ return seg;
+ }
+
+ @Override
+ int getPreInts() {
+ return LIST_PREINTS;
+ }
+
+ @Override
+ boolean isCompact() {
+ return compact;
+ }
+
+ @Override
+ boolean hasMemorySegment() {
+ return seg.scope().isAlive();
+ }
+
+ @Override
+ boolean isOffHeap() {
+ return seg.isNative();
+ }
+
+ @Override
+ boolean isSameResource(final MemorySegment seg) {
+ return MemorySegmentStatus.isSameResource(this.seg, seg);
+ }
+
+ @Override
+ PairIterator iterator() {
+ final long dataStart = getSegDataStart();
+ final int lenInts = (compact) ? getCouponCount() : 1 << getLgCouponArrInts();
+ return new IntMemorySegmentPairIterator(seg, dataStart, lenInts, lgConfigK);
+ }
+
+ @Override
+ void mergeTo(final HllSketch that) {
+ final int lenInts = (compact) ? getCouponCount() : 1 << getLgCouponArrInts();
+ final int dataStart = getSegDataStart();
+ for (int i = 0; i < lenInts; i++) {
+ final int pair = seg.get(JAVA_INT_UNALIGNED, dataStart + (i << 2));
+ if (pair == 0) { continue; }
+ that.couponUpdate(pair);
+ }
+ }
+
+ @Override
+ DirectCouponList reset() {
+ if (wseg == null) {
+ throw new SketchesArgumentException("Cannot reset a read-only sketch");
+ }
+ insertEmptyFlag(wseg, true);
+ final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ clear(wseg, 0, bytes);
+
+ return DirectCouponList.newInstance(lgConfigK, tgtHllType, wseg);
+ }
+
+ //Called by DirectCouponList.couponUpdate()
+ static final DirectCouponHashSet promoteListToSet(final DirectCouponList src) {
+ final MemorySegment wseg = src.wseg;
+
+ //get the data from the current MemorySegment
+ HllUtil.checkPreamble(wseg); //sanity check
+ final int lgConfigK = src.lgConfigK;
+ final TgtHllType tgtHllType = src.tgtHllType;
+ final int srcOffset = LIST_INT_ARR_START;
+ final int couponArrInts = 1 << src.getLgCouponArrInts();
+ final int[] couponArr = new int[couponArrInts]; //buffer
+ MemorySegment.copy(wseg, JAVA_INT_UNALIGNED, srcOffset, couponArr, 0, couponArrInts);
+
+ //rewrite the MemorySegment image as a SET:
+ insertPreInts(wseg, HASH_SET_PREINTS);
+ //SerVer, FamID, LgK should be OK
+ insertLgArr(wseg, LG_INIT_SET_SIZE);
+ insertCurMin(wseg, 0); //was list count
+ insertCurMode(wseg, CurMode.SET);
+ //tgtHllType should already be ok
+ final int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ clear(wseg, LIST_INT_ARR_START, maxBytes - LIST_INT_ARR_START); //clear all past first 8
+
+ //create the tgt
+ final DirectCouponHashSet dchSet = new DirectCouponHashSet(src.lgConfigK, src.tgtHllType, src.wseg);
+
+ //now reload the coupon data into the set
+ for (int i = 0; i < couponArrInts; i++) {
+ final int coupon = couponArr[i];
+ if (coupon != EMPTY) {
+ dchSet.couponUpdate(coupon);
+ }
+ }
+ return dchSet;
+ }
+
+ static final DirectHllArray promoteListOrSetToHll(final DirectCouponList src) {
+ final MemorySegment wseg = src.wseg;
+
+ //get the data from the current list or set MemorySegment
+ HllUtil.checkPreamble(wseg); //sanity check
+ final int lgConfigK = src.lgConfigK;
+ final TgtHllType tgtHllType = src.tgtHllType;
+ final int srcSegDataStart = src.getSegDataStart();
+ final double est = src.getEstimate();
+ final int couponArrInts = 1 << src.getLgCouponArrInts();
+ final int[] couponArr = new int[couponArrInts]; //buffer
+ MemorySegment.copy(wseg, JAVA_INT_UNALIGNED, srcSegDataStart, couponArr, 0, couponArrInts);
+
+ //rewrite the MemorySegment image as an HLL
+ insertPreInts(wseg, HLL_PREINTS);
+ //SerVer, FamID, LgK should be OK
+ insertLgArr(wseg, 0); //no Aux possible yet
+ insertFlags(wseg, 0); //clear all flags
+ insertCurMin(wseg, 0);
+ insertCurMode(wseg, CurMode.HLL);
+ //tgtHllType should already be set
+ //we update HipAccum at the end
+ //clear KxQ0, KxQ1, NumAtCurMin, AuxCount, hllArray, auxArr
+ final int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ clear(wseg, LIST_INT_ARR_START, maxBytes - LIST_INT_ARR_START); //clear all past first 8
+ insertNumAtCurMin(wseg, 1 << lgConfigK); //set numAtCurMin
+ insertKxQ0(wseg, 1 << lgConfigK);
+
+ //choose the tgt
+ final DirectHllArray dirHllArr;
+ if (tgtHllType == TgtHllType.HLL_4) {
+ dirHllArr = new DirectHll4Array(lgConfigK, wseg);
+ } else if (tgtHllType == TgtHllType.HLL_6) {
+ dirHllArr = new DirectHll6Array(lgConfigK, wseg);
+ } else { //Hll_8
+ dirHllArr = new DirectHll8Array(lgConfigK, wseg);
+ }
+
+ //now load the coupon data into HLL
+ for (int i = 0; i < couponArrInts; i++) {
+ final int coupon = couponArr[i];
+ if (coupon != EMPTY) {
+ dirHllArr.couponUpdate(coupon);
+ }
+ }
+ dirHllArr.putHipAccum(est);
+ return dirHllArr;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectHll4Array.java b/src/main/java/org/apache/datasketches/hll2/DirectHll4Array.java
new file mode 100644
index 000000000..9e3c1e415
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectHll4Array.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.apache.datasketches.hll2.HllUtil.AUX_TOKEN;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.HllUtil.hiNibbleMask;
+import static org.apache.datasketches.hll2.HllUtil.loNibbleMask;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.extractAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertInt;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+final class DirectHll4Array extends DirectHllArray {
+
+ //Called by HllSketch.writableWrap(), DirectCouponList.promoteListOrSetToHll
+ DirectHll4Array(final int lgConfigK, final MemorySegment wseg) {
+ super(lgConfigK, TgtHllType.HLL_4, wseg);
+ if (extractAuxCount(seg) > 0) {
+ putAuxHashMap(new DirectAuxHashMap(this, false), false);
+ }
+ }
+
+ //Called by HllSketch.wrap(MemorySegment)
+ DirectHll4Array(final int lgConfigK, final MemorySegment seg, final boolean readOnly) {
+ super(lgConfigK, TgtHllType.HLL_4, seg, true);
+ final int auxCount = extractAuxCount(seg);
+ if (auxCount > 0) {
+ final boolean compact = extractCompactFlag(seg);
+ final AuxHashMap auxHashMap;
+ if (compact) {
+ auxHashMap = HeapAuxHashMap.heapify(seg, auxStart, lgConfigK, auxCount, compact);
+ } else {
+ auxHashMap = new DirectAuxHashMap(this, false); //not compact
+ }
+ putAuxHashMap(auxHashMap, compact);
+ }
+ }
+
+ @Override
+ HllSketchImpl copy() {
+ return Hll4Array.heapify(seg);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ if (wseg == null) { noWriteAccess(); }
+ final int newValue = coupon >>> KEY_BITS_26;
+ final int configKmask = (1 << getLgConfigK()) - 1;
+ final int slotNo = coupon & configKmask;
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getHllByteArrBytes() {
+ return hll4ArrBytes(lgConfigK);
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ final long offset = HLL_BYTE_ARR_START + (slotNo >>> 1);
+ int theByte = seg.get(JAVA_BYTE, offset);
+ if ((slotNo & 1) > 0) { //odd?
+ theByte >>>= 4;
+ }
+ return theByte & loNibbleMask;
+ }
+
+ @Override
+ int getSlotValue(final int slotNo) {
+ final int nib = getNibble(slotNo);
+ if (nib == AUX_TOKEN) {
+ final AuxHashMap auxHashMap = getAuxHashMap();
+ return auxHashMap.mustFindValueFor(slotNo); //auxHashMap cannot be null here
+ } else {
+ return nib + getCurMin();
+ }
+ }
+
+ @Override
+ int getUpdatableSerializationBytes() {
+ final AuxHashMap auxHashMap = getAuxHashMap();
+ final int auxBytes;
+ if (auxHashMap == null) {
+ auxBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
+ } else {
+ auxBytes = 4 << auxHashMap.getLgAuxArrInts();
+ }
+ return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxBytes;
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new DirectHll4Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ final long offset = HLL_BYTE_ARR_START + (slotNo >>> 1);
+ final int oldValue = seg.get(JAVA_BYTE, offset);
+ final byte value = ((slotNo & 1) == 0) //even?
+ ? (byte) ((oldValue & hiNibbleMask) | (nibValue & loNibbleMask)) //set low nibble
+ : (byte) ((oldValue & loNibbleMask) | ((nibValue << 4) & hiNibbleMask)); //set high nibble
+ wseg.set(JAVA_BYTE, offset, value);
+ }
+
+ @Override
+ //Would be used by Union, but not used because the gadget is always HLL8 type
+ void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ Hll4Update.internalHll4Update(this, slotNo, newValue);
+ }
+
+ @Override
+ byte[] toCompactByteArray() {
+ final boolean srcSegIsCompact = extractCompactFlag(seg);
+ final int totBytes = getCompactSerializationBytes();
+ final byte[] byteArr = new byte[totBytes];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArr);
+ if (srcSegIsCompact) { //seg is already consistent with result
+ MemorySegment.copy(seg, 0, segOut, 0, totBytes);
+ return byteArr;
+ }
+ //everything but the aux array is consistent
+ MemorySegment.copy(seg, 0, segOut, 0, auxStart);
+ if (auxHashMap != null) {
+ final int auxCount = auxHashMap.getAuxCount();
+ insertAuxCount(segOut, auxCount);
+ insertLgArr(segOut, auxHashMap.getLgAuxArrInts()); //only used for direct HLL
+ final PairIterator itr = auxHashMap.getIterator();
+ int cnt = 0;
+ while (itr.nextValid()) { //works whether src has compact MemorySegment or not
+ insertInt(segOut, auxStart + (cnt++ << 2), itr.getPair());
+ }
+ assert cnt == auxCount;
+ }
+ insertCompactFlag(segOut, true);
+ return byteArr;
+ }
+
+ @Override
+ byte[] toUpdatableByteArray() {
+ final boolean segIsCompact = extractCompactFlag(seg);
+ final int totBytes = getUpdatableSerializationBytes();
+ final byte[] byteArr = new byte[totBytes];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArr);
+
+ if (!segIsCompact) { //both seg and target are updatable
+ MemorySegment.copy(seg, 0, segOut, 0, totBytes);
+
+ return byteArr;
+ }
+ //seg is compact, need to handle auxArr. Easiest way:
+ final HllSketch heapSk = HllSketch.heapify(seg);
+ return heapSk.toUpdatableByteArray();
+ }
+
+ //ITERATOR
+
+ final class DirectHll4Iterator extends HllPairIterator {
+
+ DirectHll4Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ }
+
+ @Override
+ int value() {
+ return getSlotValue(index);
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectHll6Array.java b/src/main/java/org/apache/datasketches/hll2/DirectHll6Array.java
new file mode 100644
index 000000000..165f34ac5
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectHll6Array.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED;
+import static org.apache.datasketches.hll2.HllUtil.VAL_MASK_6;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+final class DirectHll6Array extends DirectHllArray {
+
+ //Called by HllSketch.writableWrap(), DirectCouponList.promoteListOrSetToHll
+ DirectHll6Array(final int lgConfigK, final MemorySegment wseg) {
+ super(lgConfigK, TgtHllType.HLL_6, wseg);
+ }
+
+ //Called by HllSketch.wrap(MemorySegment)
+ DirectHll6Array(final int lgConfigK, final MemorySegment seg, final boolean readOnly) {
+ super(lgConfigK, TgtHllType.HLL_6, seg, true);
+ }
+
+ @Override
+ HllSketchImpl copy() {
+ return Hll6Array.heapify(seg);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ if (wseg == null) { noWriteAccess(); }
+ final int newValue = HllUtil.getPairValue(coupon);
+ final int configKmask = (1 << getLgConfigK()) - 1;
+ final int slotNo = HllUtil.getPairLow26(coupon) & configKmask;
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getHllByteArrBytes() {
+ return hll6ArrBytes(lgConfigK);
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override int getSlotValue(final int slotNo) {
+ return get6Bit(seg, HLL_BYTE_ARR_START, slotNo);
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new DirectHll6Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Would be used by Union, but not used because the gadget is always HLL8 type
+ void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ if (newValue > oldValue) {
+ put6Bit(wseg, HLL_BYTE_ARR_START, slotNo, newValue);
+ hipAndKxQIncrementalUpdate(this, oldValue, newValue);
+ if (oldValue == 0) {
+ decNumAtCurMin(); //overloaded as num zeros
+ assert getNumAtCurMin() >= 0;
+ }
+ }
+ }
+
+ //off-heap / direct
+ private static void put6Bit(final MemorySegment wseg, final int offsetBytes, final int slotNo,
+ final int newValue) {
+ final int startBit = slotNo * 6;
+ final int shift = startBit & 0X7;
+ final int byteIdx = (startBit >>> 3) + offsetBytes;
+ final int valShifted = (newValue & 0X3F) << shift;
+ final int curMasked = wseg.get(JAVA_SHORT_UNALIGNED, byteIdx) & (~(VAL_MASK_6 << shift));
+ final short insert = (short) (curMasked | valShifted);
+ wseg.set(JAVA_SHORT_UNALIGNED, byteIdx, insert);
+ }
+
+ //off-heap / direct
+ private static int get6Bit(final MemorySegment seg, final int offsetBytes, final int slotNo) {
+ final int startBit = slotNo * 6;
+ final int shift = startBit & 0X7;
+ final int byteIdx = (startBit >>> 3) + offsetBytes;
+ return (byte) ((seg.get(JAVA_SHORT_UNALIGNED, byteIdx) >>> shift) & 0X3F);
+ }
+
+ //ITERATOR
+
+ final class DirectHll6Iterator extends HllPairIterator {
+ int bitOffset;
+
+ DirectHll6Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ bitOffset = -6;
+ }
+
+ @Override
+ int value() {
+ bitOffset += 6;
+ final int tmp = seg.get(JAVA_SHORT_UNALIGNED, HLL_BYTE_ARR_START + (bitOffset / 8));
+ final int shift = (bitOffset % 8) & 0X7;
+ return (tmp >>> shift) & VAL_MASK_6;
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectHll8Array.java b/src/main/java/org/apache/datasketches/hll2/DirectHll8Array.java
new file mode 100644
index 000000000..e9d93c09d
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectHll8Array.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.apache.datasketches.hll2.HllUtil.VAL_MASK_6;
+import static org.apache.datasketches.hll2.HllUtil.noWriteAccess;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * Uses 8 bits per slot in a byte array.
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+class DirectHll8Array extends DirectHllArray {
+
+ //Called by HllSketch.writableWrap(), DirectCouponList.promoteListOrSetToHll
+ DirectHll8Array(final int lgConfigK, final MemorySegment wseg) {
+ super(lgConfigK, TgtHllType.HLL_8, wseg);
+ }
+
+ //Called by HllSketch.wrap(MemorySegment)
+ DirectHll8Array(final int lgConfigK, final MemorySegment seg, final boolean readOnly) {
+ super(lgConfigK, TgtHllType.HLL_8, seg, true);
+ }
+
+ @Override
+ HllSketchImpl copy() {
+ return Hll8Array.heapify(seg);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ if (wseg == null) { noWriteAccess(); }
+ final int newValue = HllUtil.getPairValue(coupon);
+ final int configKmask = (1 << getLgConfigK()) - 1;
+ final int slotNo = HllUtil.getPairLow26(coupon) & configKmask;
+
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getHllByteArrBytes() {
+ return hll8ArrBytes(lgConfigK);
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ final int getSlotValue(final int slotNo) {
+ return seg.get(JAVA_BYTE, HLL_BYTE_ARR_START + slotNo) & VAL_MASK_6;
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new DirectHll8Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by Union when source is not HLL8
+ final void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ if (newValue > oldValue) {
+ wseg.set(JAVA_BYTE, HLL_BYTE_ARR_START + slotNo, (byte) (newValue & VAL_MASK_6));
+ }
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ final void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ if (newValue > oldValue) {
+ wseg.set(JAVA_BYTE, HLL_BYTE_ARR_START + slotNo, (byte) (newValue & VAL_MASK_6));
+ hipAndKxQIncrementalUpdate(this, oldValue, newValue);
+ if (oldValue == 0) {
+ decNumAtCurMin();
+ assert getNumAtCurMin() >= 0;
+ }
+ }
+ }
+
+ //ITERATOR
+
+ final class DirectHll8Iterator extends HllPairIterator {
+
+ DirectHll8Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ }
+
+ @Override
+ int value() {
+ final int tmp = seg.get(JAVA_BYTE, HLL_BYTE_ARR_START + index);
+ return tmp & VAL_MASK_6;
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll2/DirectHllArray.java
new file mode 100644
index 000000000..3c6aa6355
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/DirectHllArray.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED;
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.common.Util.clear;
+import static org.apache.datasketches.hll2.PreambleUtil.CUR_MIN_COUNT_INT;
+import static org.apache.datasketches.hll2.PreambleUtil.HIP_ACCUM_DOUBLE;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCurMode;
+import static org.apache.datasketches.hll2.PreambleUtil.extractEmptyFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractHipAccum;
+import static org.apache.datasketches.hll2.PreambleUtil.extractKxQ0;
+import static org.apache.datasketches.hll2.PreambleUtil.extractKxQ1;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.extractNumAtCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.extractOooFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractRebuildCurMinNumKxQFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractTgtHllType;
+import static org.apache.datasketches.hll2.PreambleUtil.insertAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertEmptyFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertHipAccum;
+import static org.apache.datasketches.hll2.PreambleUtil.insertKxQ0;
+import static org.apache.datasketches.hll2.PreambleUtil.insertKxQ1;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.insertNumAtCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertOooFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertRebuildCurMinNumKxQFlag;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.MemorySegmentStatus;
+import org.apache.datasketches.common.SketchesArgumentException;
+
+/**
+ * @author Lee Rhodes
+ */
+abstract class DirectHllArray extends AbstractHllArray {
+ MemorySegment wseg; //used for writable direct
+ MemorySegment seg; //used for compact, read-only direct
+ Object segObj; //used temporarily for byte-array
+ final boolean compact;
+
+ private static int checkSegCompactFlag(final MemorySegment wseg, final int lgConfigK) {
+ assert !extractCompactFlag(wseg);
+ return lgConfigK;
+ }
+
+ //Memory must be already initialized and may have data. Writable, must not be Compact.
+ DirectHllArray(final int lgConfigK, final TgtHllType tgtHllType, final MemorySegment wseg) {
+ super(checkSegCompactFlag(wseg, lgConfigK), tgtHllType, CurMode.HLL);
+ this.wseg = wseg;
+ seg = wseg;
+ segObj = wseg.toArray(JAVA_BYTE);
+ compact = extractCompactFlag(seg);
+ insertEmptyFlag(wseg, false);
+ }
+
+ //Memory must already be initialized and should have data. Read-only. May be Compact or not
+ DirectHllArray(final int lgConfigK, final TgtHllType tgtHllType, final MemorySegment seg, final boolean readOnly) {
+ super(lgConfigK, tgtHllType, CurMode.HLL);
+ wseg = null;
+ this.seg = seg;
+ segObj = seg.toArray(JAVA_BYTE);
+ compact = extractCompactFlag(seg);
+ }
+
+ //only called by DirectAuxHashMap
+ final void updateMemorySegment(final MemorySegment newWseg) {
+ wseg = newWseg;
+ seg = newWseg;
+ segObj = wseg.toArray(JAVA_BYTE);
+ }
+
+ @Override
+ void addToHipAccum(final double delta) {
+ checkReadOnly(wseg);
+ final double hipAccum = seg.get(JAVA_DOUBLE_UNALIGNED, HIP_ACCUM_DOUBLE);
+ wseg.set(JAVA_DOUBLE_UNALIGNED, HIP_ACCUM_DOUBLE, hipAccum + delta);
+ }
+
+ @Override
+ void decNumAtCurMin() {
+ checkReadOnly(wseg);
+ int numAtCurMin = seg.get(JAVA_INT_UNALIGNED, CUR_MIN_COUNT_INT);
+ wseg.set(JAVA_INT_UNALIGNED, CUR_MIN_COUNT_INT, --numAtCurMin);
+ }
+
+ @Override
+ int getCurMin() {
+ return extractCurMin(seg);
+ }
+
+ @Override
+ CurMode getCurMode() {
+ return extractCurMode(seg);
+ }
+
+ @Override
+ double getHipAccum() {
+ return extractHipAccum(seg);
+ }
+
+ @Override
+ double getKxQ0() {
+ return extractKxQ0(seg);
+ }
+
+ @Override
+ double getKxQ1() {
+ return extractKxQ1(seg);
+ }
+
+ @Override
+ int getLgConfigK() {
+ return extractLgK(seg);
+ }
+
+ @Override
+ MemorySegment getMemorySegment() {
+ return seg;
+ }
+
+ @Override
+ AuxHashMap getNewAuxHashMap() {
+ return new DirectAuxHashMap(this, true);
+ }
+
+ @Override
+ int getNumAtCurMin() {
+ return extractNumAtCurMin(seg);
+ }
+
+ @Override
+ TgtHllType getTgtHllType() {
+ return extractTgtHllType(seg);
+ }
+
+ @Override
+ boolean isCompact() {
+ return compact;
+ }
+
+ @Override
+ boolean isEmpty() {
+ return extractEmptyFlag(seg);
+ }
+
+ @Override
+ boolean hasMemorySegment() {
+ return seg.scope().isAlive();
+ }
+
+ @Override
+ boolean isOffHeap() {
+ return seg.isNative();
+ }
+
+ @Override
+ boolean isOutOfOrder() {
+ return extractOooFlag(seg);
+ }
+
+ @Override
+ boolean isSameResource(final MemorySegment seg) {
+ return MemorySegmentStatus.isSameResource(this.seg, seg);
+ }
+
+ @Override
+ boolean isRebuildCurMinNumKxQFlag() {
+ return extractRebuildCurMinNumKxQFlag(seg);
+ }
+
+ @Override
+ void putAuxHashMap(final AuxHashMap auxHashMap, final boolean compact) {
+ if (auxHashMap instanceof HeapAuxHashMap) {
+ if (compact) {
+ this.auxHashMap = auxHashMap; //heap and compact
+ } else { //heap and not compact
+ final int[] auxArr = auxHashMap.getAuxIntArr();
+ MemorySegment.copy(auxArr, 0, wseg, JAVA_INT_UNALIGNED, auxStart, auxArr.length);
+ insertLgArr(wseg, auxHashMap.getLgAuxArrInts());
+ insertAuxCount(wseg, auxHashMap.getAuxCount());
+ this.auxHashMap = new DirectAuxHashMap(this, false);
+ }
+ } else { //DirectAuxHashMap
+ assert !compact; //must not be compact
+ this.auxHashMap = auxHashMap; //In case of read-only this works.
+ }
+ }
+
+ @Override
+ void putCurMin(final int curMin) {
+ checkReadOnly(wseg);
+ insertCurMin(wseg, curMin);
+ }
+
+ @Override
+ void putEmptyFlag(final boolean empty) {
+ checkReadOnly(wseg);
+ insertEmptyFlag(wseg, empty);
+ }
+
+ @Override
+ void putHipAccum(final double hipAccum) {
+ checkReadOnly(wseg);
+ insertHipAccum(wseg, hipAccum);
+ }
+
+ @Override
+ void putKxQ0(final double kxq0) {
+ checkReadOnly(wseg);
+ insertKxQ0(wseg, kxq0);
+ }
+
+ @Override //called very very very rarely
+ void putKxQ1(final double kxq1) {
+ checkReadOnly(wseg);
+ insertKxQ1(wseg, kxq1);
+ }
+
+ @Override
+ void putNumAtCurMin(final int numAtCurMin) {
+ checkReadOnly(wseg);
+ insertNumAtCurMin(wseg, numAtCurMin);
+ }
+
+ @Override //not used on the direct side
+ void putOutOfOrder(final boolean oooFlag) {
+ if (oooFlag) { putHipAccum(0); }
+ checkReadOnly(wseg);
+ insertOooFlag(wseg, oooFlag);
+ }
+
+ @Override
+ void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+ checkReadOnly(wseg);
+ insertRebuildCurMinNumKxQFlag(wseg, rebuild);
+ }
+
+ @Override //used by HLL6 and HLL8, overridden by HLL4
+ byte[] toCompactByteArray() {
+ return toUpdatableByteArray(); //indistinguishable for HLL6 and HLL8
+ }
+
+ @Override //used by HLL6 and HLL8, overridden by HLL4
+ byte[] toUpdatableByteArray() {
+ final int totBytes = getCompactSerializationBytes();
+ final byte[] byteArr = new byte[totBytes];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArr);
+ MemorySegment.copy(seg, 0, segOut, 0, totBytes);
+ insertCompactFlag(segOut, false);
+ return byteArr;
+ }
+
+ @Override
+ HllSketchImpl reset() {
+ checkReadOnly(wseg);
+ insertEmptyFlag(wseg, true);
+ final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ clear(wseg, 0, bytes);
+ return DirectCouponList.newInstance(lgConfigK, tgtHllType, wseg);
+ }
+
+ private static final void checkReadOnly(final MemorySegment wseg) {
+ if (wseg == null) {
+ throw new SketchesArgumentException("Cannot modify a read-only sketch");
+ }
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HarmonicNumbers.java b/src/main/java/org/apache/datasketches/hll2/HarmonicNumbers.java
new file mode 100644
index 000000000..db23921b3
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HarmonicNumbers.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class HarmonicNumbers {
+
+ /**
+ * This is the estimator you would use for flat bit map random accessed, similar to a Bloom filter.
+ * @param bitVectorLength the length of the bit vector in bits. Must be > 0.
+ * @param numBitsSet the number of bits set in this bit vector. Must be ≥ 0 and ≤
+ * bitVectorLength.
+ * @return the estimate.
+ */
+ static double getBitMapEstimate(final int bitVectorLength, final int numBitsSet) {
+ return (bitVectorLength * (harmonicNumber(bitVectorLength)
+ - harmonicNumber(bitVectorLength - numBitsSet))); //converts to numZeros
+ }
+
+ //In C: two-registers.c Line 1170
+ private static final int NUM_EXACT_HARMONIC_NUMBERS = 25;
+
+ private static double[] tableOfExactHarmonicNumbers = {
+ 0.0, // 0
+ 1.0, // 1
+ 1.5, // 2
+ 11.0 / 6.0, // 3
+ 25.0 / 12.0, // 4
+ 137.0 / 60.0, // 5
+ 49.0 / 20.0, // 6
+ 363.0 / 140.0, // 7
+ 761.0 / 280.0, // 8
+ 7129.0 / 2520.0, // 9
+ 7381.0 / 2520.0, // 10
+ 83711.0 / 27720.0, // 11
+ 86021.0 / 27720.0, // 12
+ 1145993.0 / 360360.0, // 13
+ 1171733.0 / 360360.0, // 14
+ 1195757.0 / 360360.0, // 15
+ 2436559.0 / 720720.0, // 16
+ 42142223.0 / 12252240.0, // 17
+ 14274301.0 / 4084080.0, // 18
+ 275295799.0 / 77597520.0, // 19
+ 55835135.0 / 15519504.0, // 20
+ 18858053.0 / 5173168.0, // 21
+ 19093197.0 / 5173168.0, // 22
+ 444316699.0 / 118982864.0, // 23
+ 1347822955.0 / 356948592.0 // 24
+ };
+
+ //In C: two-register.c Line 1202
+ private static final double EULER_MASCHERONI_CONSTANT = 0.577215664901532860606512090082;
+
+ private static double harmonicNumber(final long x_i) {
+ if (x_i < NUM_EXACT_HARMONIC_NUMBERS) {
+ return tableOfExactHarmonicNumbers[(int) x_i];
+ } else {
+ final double x = x_i;
+ final double invSq = 1.0 / (x * x);
+ double sum = Math.log(x) + EULER_MASCHERONI_CONSTANT + (1.0 / (2.0 * x));
+ /* note: the number of terms included from this series expansion is appropriate
+ for the size of the exact table (25) and the precision of doubles */
+ double pow = invSq; /* now n^-2 */
+ sum -= pow * (1.0 / 12.0);
+ pow *= invSq; /* now n^-4 */
+ sum += pow * (1.0 / 120.0);
+ pow *= invSq; /* now n^-6 */
+ sum -= pow * (1.0 / 252.0);
+ pow *= invSq; /* now n^-8 */
+ sum += pow * (1.0 / 240.0);
+ return sum;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HeapAuxHashMap.java b/src/main/java/org/apache/datasketches/hll2/HeapAuxHashMap.java
new file mode 100644
index 000000000..740d9d1f8
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HeapAuxHashMap.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_DENOM;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_NUMER;
+import static org.apache.datasketches.hll2.PreambleUtil.extractInt;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgArr;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class HeapAuxHashMap implements AuxHashMap {
+ private final int lgConfigK; //required for #slot bits
+ private int lgAuxArrInts;
+ private int auxCount;
+ private int[] auxIntArr; //used by Hll4Array
+
+ /**
+ * Standard constructor
+ * @param lgAuxArrInts the log size of the aux integer array
+ * @param lgConfigK must be 7 to 21
+ */
+ HeapAuxHashMap(final int lgAuxArrInts, final int lgConfigK) {
+ this.lgConfigK = lgConfigK;
+ this.lgAuxArrInts = lgAuxArrInts;
+ auxIntArr = new int[1 << lgAuxArrInts];
+ }
+
+ /**
+ * Copy constructor
+ * @param that another AuxHashMap
+ */
+ HeapAuxHashMap(final HeapAuxHashMap that) {
+ lgConfigK = that.lgConfigK;
+ lgAuxArrInts = that.lgAuxArrInts;
+ auxCount = that.auxCount;
+ auxIntArr = that.auxIntArr.clone();
+ }
+
+ static HeapAuxHashMap heapify(final MemorySegment seg, final long offset, final int lgConfigK,
+ final int auxCount, final boolean srcCompact) {
+ final int lgAuxArrInts;
+ final HeapAuxHashMap auxMap;
+ if (srcCompact) { //early versions did not use LgArr byte field
+ lgAuxArrInts = PreambleUtil.computeLgArr(seg, auxCount, lgConfigK);
+ } else { //updatable
+ lgAuxArrInts = extractLgArr(seg);
+ }
+ auxMap = new HeapAuxHashMap(lgAuxArrInts, lgConfigK);
+
+ final int configKmask = (1 << lgConfigK) - 1;
+
+ if (srcCompact) {
+ for (int i = 0; i < auxCount; i++) {
+ final int pair = extractInt(seg, offset + (i << 2));
+ final int slotNo = HllUtil.getPairLow26(pair) & configKmask;
+ final int value = HllUtil.getPairValue(pair);
+ auxMap.mustAdd(slotNo, value); //increments count
+ }
+ } else { //updatable
+ final int auxArrInts = 1 << lgAuxArrInts;
+ for (int i = 0; i < auxArrInts; i++) {
+ final int pair = extractInt(seg, offset + (i << 2));
+ if (pair == EMPTY) { continue; }
+ final int slotNo = HllUtil.getPairLow26(pair) & configKmask;
+ final int value = HllUtil.getPairValue(pair);
+ auxMap.mustAdd(slotNo, value); //increments count
+ }
+ }
+ return auxMap;
+ }
+
+ @Override
+ public HeapAuxHashMap copy() {
+ return new HeapAuxHashMap(this);
+ }
+
+ @Override
+ public int getAuxCount() {
+ return auxCount;
+ }
+
+ @Override
+ public int[] getAuxIntArr() {
+ return auxIntArr;
+ }
+
+ @Override
+ public int getCompactSizeBytes() {
+ return auxCount << 2;
+ }
+
+ @Override
+ public PairIterator getIterator() {
+ return new IntArrayPairIterator(auxIntArr, lgConfigK);
+ }
+
+ @Override
+ public int getLgAuxArrInts() {
+ return lgAuxArrInts;
+ }
+
+ @Override
+ public int getUpdatableSizeBytes() {
+ return 4 << lgAuxArrInts;
+ }
+
+ @Override
+ public boolean hasMemorySegment() {
+ return false;
+ }
+
+ @Override
+ public boolean isOffHeap() {
+ return false;
+ }
+
+ @Override
+ public boolean isSameResource(final MemorySegment seg) {
+ return false;
+ }
+
+ //In C: two-registers.c Line 300.
+ @Override
+ public void mustAdd(final int slotNo, final int value) {
+ final int index = find(auxIntArr, lgAuxArrInts, lgConfigK, slotNo);
+ final int pair = HllUtil.pair(slotNo, value);
+ if (index >= 0) {
+ final String pairStr = HllUtil.pairString(pair);
+ throw new SketchesStateException("Found a slotNo that should not be there: " + pairStr);
+ }
+ //Found empty entry
+ auxIntArr[~index] = pair;
+ auxCount++;
+ checkGrow();
+ }
+
+ //In C: two-registers.c Line 205
+ @Override
+ public int mustFindValueFor(final int slotNo) {
+ final int index = find(auxIntArr, lgAuxArrInts, lgConfigK, slotNo);
+ if (index >= 0) {
+ return HllUtil.getPairValue(auxIntArr[index]);
+ }
+ throw new SketchesStateException("SlotNo not found: " + slotNo);
+ }
+
+ //In C: two-registers.c Line 321.
+ @Override
+ public void mustReplace(final int slotNo, final int value) {
+ final int idx = find(auxIntArr, lgAuxArrInts, lgConfigK, slotNo);
+ if (idx >= 0) {
+ auxIntArr[idx] = HllUtil.pair(slotNo, value); //replace
+ return;
+ }
+ final String pairStr = HllUtil.pairString(HllUtil.pair(slotNo, value));
+ throw new SketchesStateException("Pair not found: " + pairStr);
+ }
+
+ //Searches the Aux arr hash table for an empty or a matching slotNo depending on the context.
+ //If entire entry is empty, returns one's complement of index = found empty.
+ //If entry contains given slotNo, returns its index = found slotNo.
+ //Continues searching.
+ //If the probe comes back to original index, throws an exception.
+ private static int find(final int[] auxArr, final int lgAuxArrInts, final int lgConfigK,
+ final int slotNo) {
+ assert lgAuxArrInts < lgConfigK;
+ final int auxArrMask = (1 << lgAuxArrInts) - 1;
+ final int configKmask = (1 << lgConfigK) - 1;
+ int probe = slotNo & auxArrMask;
+ final int loopIndex = probe;
+ do {
+ final int arrVal = auxArr[probe];
+ if (arrVal == EMPTY) { //Compares on entire entry
+ return ~probe; //empty
+ }
+ else if (slotNo == (arrVal & configKmask)) { //Compares only on slotNo
+ return probe; //found given slotNo, return probe = index into aux array
+ }
+ final int stride = (slotNo >>> lgAuxArrInts) | 1;
+ probe = (probe + stride) & auxArrMask;
+ } while (probe != loopIndex);
+ throw new SketchesArgumentException("Key not found and no empty slots!");
+ }
+
+ private void checkGrow() {
+ if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * auxIntArr.length)) {
+ growAuxSpace();
+ }
+ }
+
+ private void growAuxSpace() {
+ final int[] oldArray = auxIntArr;
+ final int configKmask = (1 << lgConfigK) - 1;
+ auxIntArr = new int[1 << ++lgAuxArrInts];
+ for (int i = 0; i < oldArray.length; i++) {
+ final int fetched = oldArray[i];
+ if (fetched != EMPTY) {
+ //find empty in new array
+ final int idx = find(auxIntArr, lgAuxArrInts, lgConfigK, fetched & configKmask);
+ auxIntArr[~idx] = fetched;
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Hll4Array.java b/src/main/java/org/apache/datasketches/hll2/Hll4Array.java
new file mode 100644
index 000000000..a80dbb9f9
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Hll4Array.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.AUX_TOKEN;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.HllUtil.hiNibbleMask;
+import static org.apache.datasketches.hll2.HllUtil.loNibbleMask;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.extractAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * Uses 4 bits per slot in a packed byte array.
+ * @author Lee Rhodes
+ */
+final class Hll4Array extends HllArray {
+
+ /**
+ * Standard constructor for new instance
+ * @param lgConfigK the configured Lg K
+ */
+ Hll4Array(final int lgConfigK) {
+ super(lgConfigK, TgtHllType.HLL_4);
+ hllByteArr = new byte[hll4ArrBytes(lgConfigK)];
+ }
+
+ /**
+ * Copy constructor
+ * @param that another Hll4Array
+ */
+ Hll4Array(final Hll4Array that) {
+ super(that);
+ }
+
+ static Hll4Array heapify(final MemorySegment seg) {
+ final int lgConfigK = extractLgK(seg);
+ final Hll4Array hll4Array = new Hll4Array(lgConfigK);
+ HllArray.extractCommonHll(seg, hll4Array);
+
+ //load AuxHashMap
+ final int auxStart = hll4Array.auxStart;
+ final int auxCount = extractAuxCount(seg);
+ final boolean compact = extractCompactFlag(seg);
+ HeapAuxHashMap auxHashMap = null;
+ if (auxCount > 0) {
+ auxHashMap = HeapAuxHashMap.heapify(seg, auxStart, lgConfigK, auxCount, compact);
+ }
+ hll4Array.putAuxHashMap(auxHashMap, false);
+ return hll4Array;
+ }
+
+ @Override
+ Hll4Array copy() {
+ return new Hll4Array(this);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ final int newValue = coupon >>> KEY_BITS_26;
+ final int configKmask = (1 << getLgConfigK()) - 1;
+ final int slotNo = coupon & configKmask;
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ int theByte = hllByteArr[slotNo >>> 1];
+ if ((slotNo & 1) > 0) { //odd?
+ theByte >>>= 4;
+ }
+ return theByte & loNibbleMask;
+ }
+
+ @Override
+ int getSlotValue(final int slotNo) {
+ final int nib = getNibble(slotNo);
+ if (nib == AUX_TOKEN) {
+ final AuxHashMap auxHashMap = getAuxHashMap();
+ return auxHashMap.mustFindValueFor(slotNo); //auxHashMap cannot be null here
+ } else {
+ return nib + getCurMin();
+ }
+ }
+
+ @Override
+ int getUpdatableSerializationBytes() {
+ final AuxHashMap auxHashMap = getAuxHashMap();
+ final int auxBytes;
+ if (auxHashMap == null) {
+ auxBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
+ } else {
+ auxBytes = 4 << auxHashMap.getLgAuxArrInts();
+ }
+ return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxBytes;
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new HeapHll4Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ final int byteno = slotNo >>> 1;
+ final int oldValue = hllByteArr[byteno];
+ if ((slotNo & 1) == 0) { // set low nibble
+ hllByteArr[byteno] = (byte) ((oldValue & hiNibbleMask) | (nibValue & loNibbleMask));
+ } else { //set high nibble
+ hllByteArr[byteno] = (byte) ((oldValue & loNibbleMask) | ((nibValue << 4) & hiNibbleMask));
+ }
+ }
+
+ @Override
+ //Would be used by Union, but not used because the gadget is always HLL8 type
+ void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ Hll4Update.internalHll4Update(this, slotNo, newValue);
+ }
+
+ @Override
+ byte[] toCompactByteArray() {
+ return ToByteArrayImpl.toHllByteArray(this, true);
+ }
+
+ //ITERATOR
+
+ final class HeapHll4Iterator extends HllPairIterator {
+
+ HeapHll4Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ }
+
+ @Override
+ int value() {
+ return getSlotValue(index);
+ }
+ }
+
+ @Override
+ MemorySegment getMemorySegment() {
+ return null;
+ }
+
+ @Override
+ boolean isSameResource(final MemorySegment seg) {
+ return false;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Hll4Update.java b/src/main/java/org/apache/datasketches/hll2/Hll4Update.java
new file mode 100644
index 000000000..ac006988d
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Hll4Update.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.AUX_TOKEN;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * Update process common to Heap Hll 4 and Direct Hll 4
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+class Hll4Update {
+
+ //Uses lgConfigK, curMin, numAtCurMin, auxMap
+ //Only called by Hll4Array and DirectHll4Array
+ //In C: two-registers.c Line 836 in "hhb_abstract_set_slot_if_new_value_bigger" non-sparse
+ static final void internalHll4Update(final AbstractHllArray host, final int slotNo,
+ final int newValue) {
+ assert ((0 <= slotNo) && (slotNo < (1 << host.getLgConfigK())));
+
+ final int curMin = host.getCurMin();
+ final int rawStoredOldNibble = host.getNibble(slotNo); //could be 0
+ final int lbOnOldValue = rawStoredOldNibble + curMin; //provable lower bound, could be 0
+
+ if (newValue <= lbOnOldValue) { return; }
+ //Thus: newValue > lbOnOldValue AND newValue > curMin
+
+ AuxHashMap auxHashMap; // = host.getAuxHashMap();
+ final int actualOldValue;
+ final int shiftedNewValue; //value - curMin
+
+ //Based on whether we have an AUX_TOKEN and whether the shiftedNewValue is greater than
+ // AUX_TOKEN, we have four cases for how to actually modify the data structure:
+ // 1. (shiftedNewValue >= AUX_TOKEN) && (rawStoredOldNibble = AUX_TOKEN) //881:
+ // The byte array already contains aux token
+ // This is the case where old and new values are both exceptions.
+ // Therefore, the 4-bit array already is AUX_TOKEN. Only need to update auxMap
+ // 2. (shiftedNewValue < AUX_TOKEN) && (rawStoredOldNibble = AUX_TOKEN) //885
+ // This is the (hypothetical) case where old value is an exception and the new one is not,
+ // which is impossible given that curMin has not changed here and the newValue > oldValue.
+ // 3. (shiftedNewValue >= AUX_TOKEN) && (rawStoredOldNibble < AUX_TOKEN) //892
+ // This is the case where the old value is not an exception and the new value is.
+ // Therefore the AUX_TOKEN must be stored in the 4-bit array and the new value
+ // added to the exception table.
+ // 4. (shiftedNewValue < AUX_TOKEN) && (rawStoredOldNibble < AUX_TOKEN) //897
+ // This is the case where neither the old value nor the new value is an exception.
+ // Therefore we just overwrite the 4-bit array with the shifted new value.
+
+ if (rawStoredOldNibble == AUX_TOKEN) { //846 Note: This is rare and really hard to test!
+ auxHashMap = host.getAuxHashMap(); //auxHashMap must already exist.
+ assert auxHashMap != null;
+ actualOldValue = auxHashMap.mustFindValueFor(slotNo);//lgtm [java/dereferenced-value-may-be-null]
+ if (newValue <= actualOldValue) { return; }
+ //We know that the array will be changed, but we haven't actually updated yet.
+ AbstractHllArray.hipAndKxQIncrementalUpdate(host, actualOldValue, newValue);
+ shiftedNewValue = newValue - curMin;
+ assert (shiftedNewValue >= 0);
+
+ if (shiftedNewValue >= AUX_TOKEN) { //CASE 1:
+ auxHashMap.mustReplace(slotNo, newValue); //lgtm [java/dereferenced-value-may-be-null]
+ }
+ //else //CASE 2: impossible
+
+ } else { //rawStoredOldNibble < AUX_TOKEN
+ actualOldValue = lbOnOldValue;
+ //We know that the array will be changed, but we haven't actually updated yet.
+ AbstractHllArray.hipAndKxQIncrementalUpdate(host, actualOldValue, newValue);
+ shiftedNewValue = newValue - curMin;
+ assert (shiftedNewValue >= 0);
+
+ if (shiftedNewValue >= AUX_TOKEN) { //CASE 3: //892
+ host.putNibble(slotNo, AUX_TOKEN);
+ auxHashMap = host.getAuxHashMap();
+ if (auxHashMap == null) {
+ auxHashMap = host.getNewAuxHashMap();
+ host.putAuxHashMap(auxHashMap, false);
+ }
+ auxHashMap.mustAdd(slotNo, newValue);
+ }
+ else { // CASE 4: //897
+ host.putNibble(slotNo, shiftedNewValue);
+ }
+ }
+
+ // We just changed the HLL array, so it might be time to change curMin
+ if (actualOldValue == curMin) { //908
+ assert (host.getNumAtCurMin() >= 1);
+ host.decNumAtCurMin();
+ while (host.getNumAtCurMin() == 0) {
+ //increases curMin by 1, and builds a new aux table,
+ // shifts values in 4-bit table, and recounts curMin
+ shiftToBiggerCurMin(host);
+ }
+ }
+ }
+
+ //This scheme only works with two double registers (2 kxq values).
+ // HipAccum, kxq0 and kxq1 remain untouched.
+ // This changes curMin, numAtCurMin, hllByteArr and auxMap.
+ //Entering this routine assumes that all slots have valid nibbles > 0 and <= 15.
+ //An AuxHashMap must exist if any values in the current hllByteArray are already 15.
+ //In C: again-two-registers.c Lines 710 "hhb_shift_to_bigger_curmin"
+ private static final void shiftToBiggerCurMin(final AbstractHllArray host) {
+ final int oldCurMin = host.getCurMin();
+ final int newCurMin = oldCurMin + 1;
+ final int lgConfigK = host.getLgConfigK();
+ final int configK = 1 << lgConfigK;
+ final int configKmask = configK - 1;
+
+ int numAtNewCurMin = 0;
+ int numAuxTokens = 0;
+
+ // Walk through the slots of 4-bit array decrementing stored values by one unless it
+ // equals AUX_TOKEN, where it is left alone but counted to be checked later.
+ // If oldStoredValue is 0 it is an error.
+ // If the decremented value is 0, we increment numAtNewCurMin.
+ // Because getNibble is masked to 4 bits oldStoredValue can never be > 15 or negative
+ for (int i = 0; i < configK; i++) { //724
+ int oldStoredNibble = host.getNibble(i);
+ if (oldStoredNibble == 0) {
+ throw new SketchesStateException("Array slots cannot be 0 at this point.");
+ }
+ if (oldStoredNibble < AUX_TOKEN) {
+ host.putNibble(i, --oldStoredNibble);
+ if (oldStoredNibble == 0) { numAtNewCurMin++; }
+ } else { //oldStoredNibble == AUX_TOKEN
+ numAuxTokens++;
+ assert host.getAuxHashMap() != null : "AuxHashMap cannot be null at this point.";
+ }
+ }
+
+ //If old AuxHashMap exists, walk through it updating some slots and build a new AuxHashMap
+ // if needed.
+ AuxHashMap newAuxMap = null;
+ final AuxHashMap oldAuxMap = host.getAuxHashMap();
+ if (oldAuxMap != null) {
+ int slotNum;
+ int oldActualVal;
+ int newShiftedVal;
+
+ final PairIterator itr = oldAuxMap.getIterator();
+ while (itr.nextValid()) {
+ slotNum = itr.getKey() & configKmask;
+ oldActualVal = itr.getValue();
+ newShiftedVal = oldActualVal - newCurMin;
+ assert newShiftedVal >= 0;
+
+ assert host.getNibble(slotNum) == AUX_TOKEN
+ : "Array slot != AUX_TOKEN: " + host.getNibble(slotNum);
+ if (newShiftedVal < AUX_TOKEN) { //756
+ assert (newShiftedVal == 14);
+ // The former exception value isn't one anymore, so it stays out of new AuxHashMap.
+ // Correct the AUX_TOKEN value in the HLL array to the newShiftedVal (14).
+ host.putNibble(slotNum, newShiftedVal);
+ numAuxTokens--;
+ }
+ else { //newShiftedVal >= AUX_TOKEN
+ // the former exception remains an exception, so must be added to the newAuxMap
+ if (newAuxMap == null) {
+ //Note: even in the direct case we use a heap aux map temporarily
+ newAuxMap = new HeapAuxHashMap(LG_AUX_ARR_INTS[lgConfigK], lgConfigK);
+ }
+ newAuxMap.mustAdd(slotNum, oldActualVal);
+ }
+ } //end scan of oldAuxMap
+ } //end if (auxHashMap != null)
+ else { //oldAuxMap == null
+ assert numAuxTokens == 0 : "auxTokens: " + numAuxTokens;
+ }
+
+ if (newAuxMap != null) {
+ assert newAuxMap.getAuxCount() == numAuxTokens : "auxCount: " + newAuxMap.getAuxCount()
+ + ", HLL tokens: " + numAuxTokens;
+ }
+ host.putAuxHashMap(newAuxMap, false); //if we are direct, this will do the right thing
+
+ host.putCurMin(newCurMin);
+ host.putNumAtCurMin(numAtNewCurMin);
+ } //end of shiftToBiggerCurMin
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Hll6Array.java b/src/main/java/org/apache/datasketches/hll2/Hll6Array.java
new file mode 100644
index 000000000..b2d567230
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Hll6Array.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.common.ByteArrayUtil.getShortLE;
+import static org.apache.datasketches.common.ByteArrayUtil.putShortLE;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.VAL_MASK_6;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * Uses 6 bits per slot in a packed byte array.
+ * @author Lee Rhodes
+ */
+class Hll6Array extends HllArray {
+
+ /**
+ * Standard constructor for new instance
+ * @param lgConfigK the configured Lg K
+ */
+ Hll6Array(final int lgConfigK) {
+ super(lgConfigK, TgtHllType.HLL_6);
+ hllByteArr = new byte[hll6ArrBytes(lgConfigK)];
+ }
+
+ /**
+ * Copy constructor
+ * @param that another Hll6Array
+ */
+ Hll6Array(final Hll6Array that) {
+ super(that);
+ }
+
+ static final Hll6Array heapify(final MemorySegment seg) {
+ final int lgConfigK = extractLgK(seg);
+ final Hll6Array hll6Array = new Hll6Array(lgConfigK);
+ HllArray.extractCommonHll(seg, hll6Array);
+ return hll6Array;
+ }
+
+ @Override
+ Hll6Array copy() {
+ return new Hll6Array(this);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ final int newValue = coupon >>> KEY_BITS_26;
+ final int configKmask = (1 << lgConfigK) - 1;
+ final int slotNo = coupon & configKmask;
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ final int getSlotValue(final int slotNo) {
+ return get6Bit(hllByteArr, 0, slotNo);
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new HeapHll6Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Would be used by Union, but not used because the gadget is always HLL8 type
+ final void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ final void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ if (newValue > oldValue) {
+ put6Bit(hllByteArr, 0, slotNo, newValue);
+ hipAndKxQIncrementalUpdate(this, oldValue, newValue);
+ if (oldValue == 0) {
+ numAtCurMin--; //interpret numAtCurMin as num Zeros
+ assert getNumAtCurMin() >= 0;
+ }
+ }
+ }
+
+ //on-heap
+ private static final void put6Bit(final byte[] arr, final int offsetBytes, final int slotNo,
+ final int newValue) {
+ final int startBit = slotNo * 6;
+ final int shift = startBit & 0X7;
+ final int byteIdx = (startBit >>> 3) + offsetBytes;
+ final int valShifted = (newValue & 0X3F) << shift;
+ final int curMasked = getShortLE(arr, byteIdx) & (~(VAL_MASK_6 << shift));
+ final short insert = (short) (curMasked | valShifted);
+ putShortLE(arr, byteIdx, insert);
+ }
+
+ //on-heap
+ private static final int get6Bit(final byte[] arr, final int offsetBytes, final int slotNo) {
+ final int startBit = slotNo * 6;
+ final int shift = startBit & 0X7;
+ final int byteIdx = (startBit >>> 3) + offsetBytes;
+ return (byte) ((getShortLE(arr, byteIdx) >>> shift) & 0X3F);
+ }
+
+ //ITERATOR
+
+ private final class HeapHll6Iterator extends HllPairIterator {
+ int bitOffset;
+
+ HeapHll6Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ bitOffset = - 6;
+ }
+
+ @Override
+ int value() {
+ bitOffset += 6;
+ final int tmp = getShortLE(hllByteArr, bitOffset / 8);
+ final int shift = (bitOffset % 8) & 0X7;
+ return (tmp >>> shift) & VAL_MASK_6;
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Hll8Array.java b/src/main/java/org/apache/datasketches/hll2/Hll8Array.java
new file mode 100644
index 000000000..3b8d9df19
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Hll8Array.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.VAL_MASK_6;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * Uses 8 bits per slot in a byte array.
+ * @author Lee Rhodes
+ */
+class Hll8Array extends HllArray {
+
+ /**
+ * Standard constructor for new instance
+ * @param lgConfigK the configured Lg K
+ */
+ Hll8Array(final int lgConfigK) {
+ super(lgConfigK, TgtHllType.HLL_8);
+ hllByteArr = new byte[hll8ArrBytes(lgConfigK)];
+ }
+
+ /**
+ * Copy constructor
+ * @param that another Hll8Array
+ */
+ Hll8Array(final Hll8Array that) {
+ super(that);
+ }
+
+ static final Hll8Array heapify(final MemorySegment seg) {
+ final int lgConfigK = extractLgK(seg);
+ final Hll8Array hll8Array = new Hll8Array(lgConfigK);
+ HllArray.extractCommonHll(seg, hll8Array);
+ return hll8Array;
+ }
+
+ @Override
+ Hll8Array copy() {
+ return new Hll8Array(this);
+ }
+
+ @Override
+ HllSketchImpl couponUpdate(final int coupon) {
+ final int newValue = coupon >>> KEY_BITS_26;
+ final int configKmask = (1 << lgConfigK) - 1;
+ final int slotNo = coupon & configKmask;
+ updateSlotWithKxQ(slotNo, newValue);
+ return this;
+ }
+
+ @Override
+ int getNibble(final int slotNo) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ final int getSlotValue(final int slotNo) {
+ return hllByteArr[slotNo] & VAL_MASK_6;
+ }
+
+ @Override
+ PairIterator iterator() {
+ return new HeapHll8Iterator(1 << lgConfigK);
+ }
+
+ @Override
+ void putNibble(final int slotNo, final int nibValue) {
+ throw new SketchesStateException("Improper access.");
+ }
+
+ @Override
+ //Used by Union when source is not HLL8
+ final void updateSlotNoKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ hllByteArr[slotNo] = (byte) Math.max(newValue, oldValue);
+ }
+
+ @Override
+ //Used by this couponUpdate()
+ //updates HipAccum, CurMin, NumAtCurMin, KxQs and checks newValue > oldValue
+ final void updateSlotWithKxQ(final int slotNo, final int newValue) {
+ final int oldValue = getSlotValue(slotNo);
+ if (newValue > oldValue) {
+ hllByteArr[slotNo] = (byte) (newValue & VAL_MASK_6);
+ hipAndKxQIncrementalUpdate(this, oldValue, newValue);
+ if (oldValue == 0) {
+ numAtCurMin--; //interpret numAtCurMin as num Zeros
+ assert getNumAtCurMin() >= 0;
+ }
+ }
+ }
+
+ //ITERATOR
+
+ final class HeapHll8Iterator extends HllPairIterator {
+
+ HeapHll8Iterator(final int lengthPairs) {
+ super(lengthPairs);
+ }
+
+ @Override
+ int value() {
+ return hllByteArr[index] & VAL_MASK_6;
+ }
+
+ @Override
+ public boolean nextValid() {
+ while (++index < lengthPairs) {
+ value = hllByteArr[index] & VAL_MASK_6;
+ if (value != EMPTY) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllArray.java b/src/main/java/org/apache/datasketches/hll2/HllArray.java
new file mode 100644
index 000000000..a0e4183df
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllArray.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.extractEmptyFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractHipAccum;
+import static org.apache.datasketches.hll2.PreambleUtil.extractKxQ0;
+import static org.apache.datasketches.hll2.PreambleUtil.extractKxQ1;
+import static org.apache.datasketches.hll2.PreambleUtil.extractNumAtCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.extractOooFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractRebuildCurMinNumKxQFlag;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_4;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_6;
+
+import java.lang.foreign.MemorySegment;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+abstract class HllArray extends AbstractHllArray {
+ boolean oooFlag = false; //Out-Of-Order Flag
+ boolean rebuildCurMinNumKxQ = false;
+ int curMin; //always zero for Hll6 and Hll8, only used by Hll4Array
+ int numAtCurMin; //# of values at curMin. If curMin = 0, it is # of zeros
+ double hipAccum;
+ double kxq0;
+ double kxq1;
+ byte[] hllByteArr = null; //init by sub-classes
+
+ /**
+ * Standard constructor for new instance
+ * @param lgConfigK the configured Lg K
+ * @param tgtHllType the type of target HLL sketch
+ */
+ HllArray(final int lgConfigK, final TgtHllType tgtHllType) {
+ super(lgConfigK, tgtHllType, CurMode.HLL);
+ curMin = 0;
+ numAtCurMin = 1 << lgConfigK;
+ hipAccum = 0;
+ kxq0 = 1 << lgConfigK;
+ kxq1 = 0;
+ }
+
+ /**
+ * Copy constructor
+ * @param that another HllArray
+ */
+ HllArray(final HllArray that) {
+ super(that.getLgConfigK(), that.getTgtHllType(), CurMode.HLL);
+ oooFlag = that.isOutOfOrder();
+ rebuildCurMinNumKxQ = that.isRebuildCurMinNumKxQFlag();
+ curMin = that.getCurMin();
+ numAtCurMin = that.getNumAtCurMin();
+ hipAccum = that.getHipAccum();
+ kxq0 = that.getKxQ0();
+ kxq1 = that.getKxQ1();
+ hllByteArr = that.hllByteArr.clone(); //that.hllByteArr should never be null.
+ final AuxHashMap thatAuxMap = that.getAuxHashMap();
+ if (thatAuxMap != null) {
+ putAuxHashMap(thatAuxMap.copy(), false);
+ } else {
+ putAuxHashMap(null, false);
+ }
+ }
+
+ static final HllArray newHeapHll(final int lgConfigK, final TgtHllType tgtHllType) {
+ if (tgtHllType == HLL_4) { return new Hll4Array(lgConfigK); }
+ if (tgtHllType == HLL_6) { return new Hll6Array(lgConfigK); }
+ return new Hll8Array(lgConfigK);
+ }
+
+ @Override
+ void addToHipAccum(final double delta) {
+ hipAccum += delta;
+ }
+
+ @Override
+ void decNumAtCurMin() {
+ numAtCurMin--;
+ }
+
+ @Override
+ int getCurMin() {
+ return curMin;
+ }
+
+ @Override
+ CurMode getCurMode() {
+ return curMode;
+ }
+
+ @Override
+ double getHipAccum() {
+ return hipAccum;
+ }
+
+ @Override
+ int getHllByteArrBytes() {
+ return hllByteArr.length;
+ }
+
+ @Override
+ double getKxQ0() {
+ return kxq0;
+ }
+
+ @Override
+ double getKxQ1() {
+ return kxq1;
+ }
+
+ @Override
+ int getLgConfigK() {
+ return lgConfigK;
+ }
+
+ @Override
+ MemorySegment getMemorySegment() {
+ return null;
+ }
+
+ @Override
+ AuxHashMap getNewAuxHashMap() {
+ return new HeapAuxHashMap(LG_AUX_ARR_INTS[lgConfigK], lgConfigK);
+ }
+
+ @Override
+ int getNumAtCurMin() {
+ return numAtCurMin;
+ }
+
+ @Override
+ boolean isCompact() {
+ return false;
+ }
+
+ @Override
+ boolean isEmpty() {
+ return false; //because there should be no normal way to create an HllArray that is empty
+ }
+
+ @Override
+ boolean hasMemorySegment() {
+ return false;
+ }
+
+ @Override
+ boolean isOffHeap() {
+ return false;
+ }
+
+ @Override
+ boolean isOutOfOrder() {
+ return oooFlag;
+ }
+
+ @Override
+ boolean isSameResource(final MemorySegment seg) {
+ return false;
+ }
+
+ @Override
+ boolean isRebuildCurMinNumKxQFlag() {
+ return rebuildCurMinNumKxQ;
+ }
+
+ @Override
+ void putAuxHashMap(final AuxHashMap auxHashMap, final boolean compact) {
+ this.auxHashMap = auxHashMap;
+ }
+
+ @Override
+ void putCurMin(final int curMin) {
+ this.curMin = curMin;
+ }
+
+ @Override
+ void putEmptyFlag(final boolean empty) { }
+
+ @Override
+ void putHipAccum(final double value) {
+ hipAccum = value;
+ }
+
+ @Override
+ void putKxQ0(final double kxq0) {
+ this.kxq0 = kxq0;
+ }
+
+ @Override
+ void putKxQ1(final double kxq1) {
+ this.kxq1 = kxq1;
+ }
+
+ @Override
+ void putNumAtCurMin(final int numAtCurMin) {
+ this.numAtCurMin = numAtCurMin;
+ }
+
+ @Override
+ void putOutOfOrder(final boolean oooFlag) {
+ if (oooFlag) { putHipAccum(0); }
+ this.oooFlag = oooFlag;
+ }
+
+ @Override
+ void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+ rebuildCurMinNumKxQ = rebuild;
+ }
+
+ @Override
+ HllSketchImpl reset() {
+ return new CouponList(lgConfigK, tgtHllType, CurMode.LIST);
+ }
+
+ @Override //used by HLL6 and HLL8, overridden by HLL4
+ byte[] toCompactByteArray() {
+ return toUpdatableByteArray(); //indistinguishable for HLL6 and HLL8
+ }
+
+ @Override //used by HLL4, HLL6 and HLL8
+ byte[] toUpdatableByteArray() {
+ return ToByteArrayImpl.toHllByteArray(this, false);
+ }
+
+ //used by heapify by all Heap HLL
+ static final void extractCommonHll(final MemorySegment srcSeg, final HllArray hllArray) {
+ hllArray.putOutOfOrder(extractOooFlag(srcSeg));
+ hllArray.putEmptyFlag(extractEmptyFlag(srcSeg));
+ hllArray.putCurMin(extractCurMin(srcSeg));
+ hllArray.putHipAccum(extractHipAccum(srcSeg));
+ hllArray.putKxQ0(extractKxQ0(srcSeg));
+ hllArray.putKxQ1(extractKxQ1(srcSeg));
+ hllArray.putNumAtCurMin(extractNumAtCurMin(srcSeg));
+ hllArray.putRebuildCurMinNumKxQFlag(extractRebuildCurMinNumKxQFlag(srcSeg));
+
+ //load Hll array
+ MemorySegment.copy(srcSeg, JAVA_BYTE, HLL_BYTE_ARR_START, hllArray.hllByteArr, 0, hllArray.hllByteArr.length);
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllEstimators.java b/src/main/java/org/apache/datasketches/hll2/HllEstimators.java
new file mode 100644
index 000000000..02acbfb97
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllEstimators.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.MIN_LOG_K;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+class HllEstimators {
+
+ //HLL UPPER AND LOWER BOUNDS
+
+ /*
+ * The upper and lower bounds are not symmetric and thus are treated slightly differently.
+ * For the lower bound, when the unique count is <= k, LB >= numNonZeros, where
+ * numNonZeros = k - numAtCurMin AND curMin == 0.
+ *
+ * For HLL6 and HLL8, curMin is always 0 and numAtCurMin is initialized to k and is decremented
+ * down for each valid update until it reaches 0, where it stays. Thus, for these two
+ * isomorphs, when numAtCurMin = 0, means the true curMin is > 0 and the unique count must be
+ * greater than k.
+ *
+ * HLL4 always maintains both curMin and numAtCurMin dynamically. Nonetheless, the rules for
+ * the very small values <= k where curMin = 0 still apply.
+ */
+
+ static final double hllLowerBound(final AbstractHllArray absHllArr, final int numStdDev) {
+ final int lgConfigK = absHllArr.lgConfigK;
+ final int configK = 1 << lgConfigK;
+ final double numNonZeros =
+ (absHllArr.getCurMin() == 0) ? configK - absHllArr.getNumAtCurMin() : configK;
+ final double estimate = absHllArr.getEstimate();
+ final boolean oooFlag = absHllArr.isOutOfOrder();
+ final double relErr = BaseHllSketch.getRelErr(false, oooFlag, lgConfigK, numStdDev);
+ return Math.max(estimate / (1.0 + relErr), numNonZeros);
+ }
+
+ static final double hllUpperBound(final AbstractHllArray absHllArr, final int numStdDev) {
+ final int lgConfigK = absHllArr.lgConfigK;
+ final double estimate = absHllArr.getEstimate();
+ final boolean oooFlag = absHllArr.isOutOfOrder();
+ final double relErr = BaseHllSketch.getRelErr(true, oooFlag, lgConfigK, numStdDev);
+ return estimate / (1.0 - relErr);
+ }
+
+ //THE HLL COMPOSITE ESTIMATOR
+
+ /**
+ * This is the (non-HIP) estimator.
+ * It is called "composite" because multiple estimators are pasted together.
+ * @param absHllArr an instance of the AbstractHllArray class.
+ * @return the composite estimate
+ */
+ //In C: again-two-registers.c hhb_get_composite_estimate L1489
+ static final double hllCompositeEstimate(final AbstractHllArray absHllArr) {
+ final int lgConfigK = absHllArr.getLgConfigK();
+ final double rawEst = getHllRawEstimate(lgConfigK, absHllArr.getKxQ0() + absHllArr.getKxQ1());
+
+ final double[] xArr = CompositeInterpolationXTable.xArrs[lgConfigK - MIN_LOG_K];
+ final double yStride = CompositeInterpolationXTable.yStrides[lgConfigK - MIN_LOG_K];
+ final int xArrLen = xArr.length;
+
+ if (rawEst < xArr[0]) { return 0; }
+
+ final int xArrLenM1 = xArrLen - 1;
+
+ if (rawEst > xArr[xArrLenM1]) {
+ final double finalY = yStride * (xArrLenM1);
+ final double factor = finalY / xArr[xArrLenM1];
+ return rawEst * factor;
+ }
+
+ final double adjEst =
+ CubicInterpolation.usingXArrAndYStride(xArr, yStride, rawEst);
+
+ // We need to completely avoid the linear_counting estimator if it might have a crazy value.
+ // Empirical evidence suggests that the threshold 3*k will keep us safe if 2^4 <= k <= 2^21.
+
+ if (adjEst > (3 << lgConfigK)) { return adjEst; }
+ //Alternate call
+ //if ((adjEst > (3 << lgConfigK)) || ((curMin != 0) || (numAtCurMin == 0)) ) { return adjEst; }
+
+ final double linEst =
+ getHllBitMapEstimate(lgConfigK, absHllArr.getCurMin(), absHllArr.getNumAtCurMin());
+
+ // Bias is created when the value of an estimator is compared with a threshold to decide whether
+ // to use that estimator or a different one.
+ // We conjecture that less bias is created when the average of the two estimators
+ // is compared with the threshold. Empirical measurements support this conjecture.
+
+ final double avgEst = (adjEst + linEst) / 2.0;
+
+ // The following constants comes from empirical measurements of the crossover point
+ // between the average error of the linear estimator and the adjusted HLL estimator
+ double crossOver = 0.64;
+ if (lgConfigK == 4) { crossOver = 0.718; }
+ else if (lgConfigK == 5) { crossOver = 0.672; }
+
+ return (avgEst > (crossOver * (1 << lgConfigK))) ? adjEst : linEst;
+ }
+
+ /**
+ * Estimator when N is small, roughly less than k log(k).
+ * Refer to Wikipedia: Coupon Collector Problem
+ * @param lgConfigK the current configured lgK of the sketch
+ * @param curMin the current minimum value of the HLL window
+ * @param numAtCurMin the current number of slots with the value curMin
+ * @return the very low range estimate
+ */
+ //In C: again-two-registers.c hhb_get_improved_linear_counting_estimate L1274
+ private static final double getHllBitMapEstimate(
+ final int lgConfigK, final int curMin, final int numAtCurMin) {
+ final int configK = 1 << lgConfigK;
+ final int numUnhitBuckets = (curMin == 0) ? numAtCurMin : 0;
+
+ //This will eventually go away.
+ if (numUnhitBuckets == 0) {
+ return configK * Math.log(configK / 0.5);
+ }
+
+ final int numHitBuckets = configK - numUnhitBuckets;
+ return HarmonicNumbers.getBitMapEstimate(configK, numHitBuckets);
+ }
+
+ //In C: again-two-registers.c hhb_get_raw_estimate L1167
+ //This algorithm is from Flajolet's, et al, 2007 HLL paper, Fig 3.
+ private static final double getHllRawEstimate(final int lgConfigK, final double kxqSum) {
+ final int configK = 1 << lgConfigK;
+ final double correctionFactor;
+ if (lgConfigK == 4) { correctionFactor = 0.673; }
+ else if (lgConfigK == 5) { correctionFactor = 0.697; }
+ else if (lgConfigK == 6) { correctionFactor = 0.709; }
+ else { correctionFactor = 0.7213 / (1.0 + (1.079 / configK)); }
+ return (correctionFactor * configK * configK) / kxqSum;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllPairIterator.java b/src/main/java/org/apache/datasketches/hll2/HllPairIterator.java
new file mode 100644
index 000000000..f885378ea
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllPairIterator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.pair;
+
+/**
+ * Iterates over an on-heap HLL byte array producing pairs of index, value.
+ *
+ * @author Lee Rhodes
+ */
+abstract class HllPairIterator extends PairIterator {
+ final int lengthPairs;
+ int index;
+ int value;
+
+ //Used by Direct<4,6,8>Array, Heap<4,6,8>Array
+ HllPairIterator(final int lengthPairs) {
+ this.lengthPairs = lengthPairs;
+ index = - 1;
+ }
+
+ @Override
+ public String getHeader() {
+ return String.format("%10s%6s", "Slot", "Value");
+ }
+
+ @Override
+ public int getIndex() {
+ return index;
+ }
+
+ @Override
+ public int getKey() {
+ return index;
+ }
+
+ @Override
+ public int getPair() {
+ return pair(index, value);
+ }
+
+ @Override
+ public int getSlot() {
+ return index;
+ }
+
+ @Override
+ public String getString() {
+ final int slot = getSlot();
+ final int value = getValue();
+ return String.format("%10d%6d", slot, value);
+ }
+
+ @Override
+ public int getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean nextAll() {
+ if (++index < lengthPairs) {
+ value = value();
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean nextValid() {
+ while (++index < lengthPairs) {
+ value = value();
+ if (value != EMPTY) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ abstract int value();
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllSketch.java b/src/main/java/org/apache/datasketches/hll2/HllSketch.java
new file mode 100644
index 000000000..7b12af019
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllSketch.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.common.Util.LS;
+import static org.apache.datasketches.common.Util.checkBounds;
+import static org.apache.datasketches.common.Util.clear;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.KEY_BITS_26;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.HllUtil.checkPreamble;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.extractLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.extractTgtHllType;
+
+import java.lang.foreign.MemorySegment;
+import java.util.Objects;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+
+/**
+ * The HllSketch is actually a collection of compact implementations of Phillipe Flajolet’s HyperLogLog (HLL)
+ * sketch but with significantly improved error behavior and excellent speed performance.
+ *
+ * <p>If the use case for sketching is primarily counting uniques and merging, the HLL sketch is the 2nd highest
+ * performing in terms of accuracy for storage space consumed in the DataSketches library
+ * (the new CPC sketch developed by Kevin J. Lang now beats HLL in terms of accuracy / space).
+ * For large counts, HLL sketches can be 2 to 8 times smaller for the same accuracy than the DataSketches Theta
+ * Sketches when serialized, but the Theta sketches can do set intersections and differences while HLL and CPC cannot.
+ * The CPC sketch and HLL share similar use cases, but the CPC sketch is about 30 to 40% smaller than the HLL sketch
+ * when serialized and larger than the HLL when active in a MemorySegment. Choose your weapons!</p>
+ *
+ * <p>A new HLL sketch is created with a simple constructor:</p>
+ * <pre>{@code
+ * int lgK = 12; //This is log-base2 of k, so k = 4096. lgK can be from 4 to 21
+ * HllSketch sketch = new HllSketch(lgK); //TgtHllType.HLL_4 is the default
+ * //OR
+ * HllSketch sketch = new HllSketch(lgK, TgtHllType.HLL_6);
+ * //OR
+ * HllSketch sketch = new HllSketch(lgK, TgtHllType.HLL_8);
+ * }</pre>
+ *
+ * <p>All three different sketch types are targets in that the sketches start out in a warm-up mode that is small in
+ * size and gradually grows as needed until the full HLL array is allocated. The HLL_4, HLL_6 and HLL_8 represent
+ * different levels of compression of the final HLL array where the 4, 6 and 8 refer to the number of bits each
+ * bucket of the HLL array is compressed down to.
+ * The HLL_4 is the most compressed but generally slower than the other two, especially during union operations.</p>
+ *
+ * <p>All three types share the same API. Updating the HllSketch is very simple:</p>
+ *
+ * <pre>{@code
+ * long n = 1000000;
+ * for (int i = 0; i < n; i++) {
+ * sketch.update(i);
+ * }
+ * }</pre>
+ *
+ * <p>Each of the presented integers above are first hashed into 128-bit hash values that are used by the sketch
+ * HLL algorithm, so the above loop is essentially equivalent to using a random number generator initialized with a
+ * seed so that the sequence is deterministic and random.</p>
+ *
+ * <p>Obtaining the cardinality results from the sketch is also simple:</p>
+ *
+ * <pre>{@code
+ * double estimate = sketch.getEstimate();
+ * double estUB = sketch.getUpperBound(1.0); //the upper bound at 1 standard deviation.
+ * double estLB = sketch.getLowerBound(1.0); //the lower bound at 1 standard deviation.
+ * //OR
+ * System.out.println(sketch.toString()); //will output a summary of the sketch.
+ * }</pre>
+ *
+ * <p>Which produces a console output something like this:</p>
+ *
+ * <pre>{@code
+ * ### HLL SKETCH SUMMARY:
+ * Log Config K : 12
+ * Hll Target : HLL_4
+ * Current Mode : HLL
+ * LB : 977348.7024560181
+ * Estimate : 990116.6007366662
+ * UB : 1003222.5095308956
+ * OutOfOrder Flag: false
+ * CurMin : 5
+ * NumAtCurMin : 1
+ * HipAccum : 990116.6007366662
+ * }</pre>
+ *
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+public class HllSketch extends BaseHllSketch {
+
+ /**
+ * The default Log_base2 of K
+ */
+ public static final int DEFAULT_LG_K = 12;
+
+ /**
+ * The default HLL-TYPE is HLL_4
+ */
+ public static final TgtHllType DEFAULT_HLL_TYPE = TgtHllType.HLL_4;
+
+ HllSketchImpl hllSketchImpl = null;
+
+ /**
+ * Constructs a new on-heap sketch with the default lgConfigK and tgtHllType.
+ */
+ public HllSketch() {
+ this(DEFAULT_LG_K, DEFAULT_HLL_TYPE);
+ }
+
+ /**
+ * Constructs a new on-heap sketch with the default tgtHllType.
+ * @param lgConfigK The Log2 of K for the target HLL sketch. This value must be
+ * between 4 and 21 inclusively.
+ */
+ public HllSketch(final int lgConfigK) {
+ this(lgConfigK, DEFAULT_HLL_TYPE);
+ }
+
+ /**
+ * Constructs a new on-heap sketch with the type of HLL sketch to configure.
+ * @param lgConfigK The Log2 of K for the target HLL sketch. This value must be
+ * between 4 and 21 inclusively.
+ * @param tgtHllType the desired HLL type.
+ */
+ public HllSketch(final int lgConfigK, final TgtHllType tgtHllType) {
+ hllSketchImpl = new CouponList(HllUtil.checkLgK(lgConfigK), tgtHllType, CurMode.LIST);
+ }
+
+ /**
+ * Constructs a new sketch with the type of HLL sketch to configure and the given
+ * MemorySegment as the destination for the sketch. This MemorySegment is usually configured
+ * for off-heap MemorySegment. What remains on the java heap is a thin wrapper object that reads and
+ * writes to the given MemorySegment.
+ *
+ * <p>The given <i>dstSeg</i> is checked for the required capacity as determined by
+ * {@link #getMaxUpdatableSerializationBytes(int, TgtHllType)}.
+ * @param lgConfigK The Log2 of K for the target HLL sketch. This value must be
+ * between 4 and 21 inclusively.
+ * @param tgtHllType the desired HLL type.
+ * @param dstSeg the destination MemorySegment for the sketch.
+ */
+ public HllSketch(final int lgConfigK, final TgtHllType tgtHllType, final MemorySegment dstSeg) {
+ Objects.requireNonNull(dstSeg, "Destination MemorySegment must not be null");
+ final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ final long capBytes = dstSeg.byteSize();
+ HllUtil.checkSegSize(minBytes, capBytes);
+ clear(dstSeg, 0, minBytes);
+ hllSketchImpl = DirectCouponList.newInstance(lgConfigK, tgtHllType, dstSeg);
+ }
+
+ /**
+ * Copy constructor used by copy().
+ * @param that another HllSketch
+ */
+ HllSketch(final HllSketch that) {
+ hllSketchImpl = that.hllSketchImpl.copy();
+ }
+
+ /**
+ * Special constructor used by copyAs, heapify
+ * @param that another HllSketchImpl, which must already be a copy
+ */
+ HllSketch(final HllSketchImpl that) {
+ hllSketchImpl = that;
+ }
+
+ /**
+ * Heapify the given byte array, which must be a valid HllSketch image and may have data.
+ * @param byteArray the given byte array. This byteArray is not modified and is not retained
+ * by the on-heap sketch.
+ * @return an HllSketch on the java heap.
+ */
+ public static final HllSketch heapify(final byte[] byteArray) {
+ return heapify(MemorySegment.ofArray(byteArray));
+ }
+
+ /**
+ * Heapify the given MemorySegment, which must be a valid HllSketch image and may have data.
+ * @param srcSeg the given MemorySegment, which is read-only.
+ * @return an HllSketch on the java heap.
+ */
+ public static final HllSketch heapify(final MemorySegment srcSeg) {
+ return heapify(srcSeg, true);
+ }
+
+ //used by union and above
+ static final HllSketch heapify(final MemorySegment srcSeg, final boolean checkRebuild) {
+ Objects.requireNonNull(srcSeg, "Source MemorySegment must not be null");
+ checkBounds(0, 8, srcSeg.byteSize()); //need min 8 bytes
+ final CurMode curMode = checkPreamble(srcSeg);
+ final HllSketch heapSketch;
+ if (curMode == CurMode.HLL) {
+ final TgtHllType tgtHllType = extractTgtHllType(srcSeg);
+ if (tgtHllType == TgtHllType.HLL_4) {
+ heapSketch = new HllSketch(Hll4Array.heapify(srcSeg));
+ } else if (tgtHllType == TgtHllType.HLL_6) {
+ heapSketch = new HllSketch(Hll6Array.heapify(srcSeg));
+ } else { //Hll_8
+ heapSketch = new HllSketch(Hll8Array.heapify(srcSeg));
+ if (checkRebuild) {
+ Union.checkRebuildCurMinNumKxQ(heapSketch);
+ }
+ }
+ } else if (curMode == CurMode.LIST) {
+ heapSketch = new HllSketch(CouponList.heapifyList(srcSeg));
+ } else {
+ heapSketch = new HllSketch(CouponHashSet.heapifySet(srcSeg));
+ }
+ return heapSketch;
+ }
+
+ /**
+ * Wraps the given MemorySegment, which must be a image of a valid updatable sketch,
+ * and may have data. What remains on the java heap is a
+ * thin wrapper object that reads and writes to the given MemorySegment, which, depending on
+ * how the user configures the MemorySegment, may actually reside on the Java heap or off-heap.
+ *
+ * <p>The given <i>dstSeg</i> is checked for the required capacity as determined by
+ * {@link #getMaxUpdatableSerializationBytes(int, TgtHllType)}.
+ * @param srcWseg an writable image of a valid source sketch with data.
+ * @return an HllSketch where the sketch data is in the given dstSeg.
+ */
+ public static final HllSketch writableWrap(final MemorySegment srcWseg) {
+ if (srcWseg.isReadOnly()) { return wrap(srcWseg); }
+ return writableWrap(srcWseg, true);
+ }
+
+ //used by union and above
+ static final HllSketch writableWrap( final MemorySegment srcWseg, final boolean checkRebuild) {
+ Objects.requireNonNull(srcWseg, "Source MemorySegment must not be null");
+ checkBounds(0, 8, srcWseg.byteSize()); //need min 8 bytes
+ if (extractCompactFlag(srcWseg)) {
+ throw new SketchesArgumentException(
+ "Cannot perform a writableWrap of a writable sketch image that is in compact form. "
+ + "Compact sketches are by definition immutable.");
+ }
+ final int lgConfigK = extractLgK(srcWseg);
+ final TgtHllType tgtHllType = extractTgtHllType(srcWseg);
+ final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ final long capBytes = srcWseg.byteSize();
+ HllUtil.checkSegSize(minBytes, capBytes);
+ final CurMode curMode = checkPreamble(srcWseg);
+ final HllSketch directSketch;
+ if (curMode == CurMode.HLL) {
+ if (tgtHllType == TgtHllType.HLL_4) {
+ directSketch = new HllSketch(new DirectHll4Array(lgConfigK, srcWseg));
+ } else if (tgtHllType == TgtHllType.HLL_6) {
+ directSketch = new HllSketch(new DirectHll6Array(lgConfigK, srcWseg));
+ } else { //Hll_8
+ directSketch = new HllSketch(new DirectHll8Array(lgConfigK, srcWseg));
+ if (checkRebuild) { //union only uses HLL_8, we allow non-finalized from a union call.
+ Union.checkRebuildCurMinNumKxQ(directSketch);
+ }
+ }
+ } else if (curMode == CurMode.LIST) {
+ directSketch =
+ new HllSketch(new DirectCouponList(lgConfigK, tgtHllType, curMode, srcWseg));
+ } else { //SET
+ directSketch =
+ new HllSketch(new DirectCouponHashSet(lgConfigK, tgtHllType, srcWseg));
+ }
+ return directSketch;
+ }
+
+ /**
+ * Wraps the given read-only MemorySegment that must be a image of a valid sketch,
+ * which may be in compact or updatable form, and should have data. Any attempt to update the
+ * given source MemorySegment will throw an exception.
+ * @param srcSeg a read-only image of a valid source sketch.
+ * @return an HllSketch, where the read-only data of the sketch is in the given srcSeg.
+ *
+ */
+ public static final HllSketch wrap(final MemorySegment srcSeg) { //read only
+ Objects.requireNonNull(srcSeg, "Source MemorySegment must not be null");
+ checkBounds(0, 8, srcSeg.byteSize()); //need min 8 bytes
+ final int lgConfigK = extractLgK(srcSeg);
+ final TgtHllType tgtHllType = extractTgtHllType(srcSeg);
+
+ final CurMode curMode = checkPreamble(srcSeg);
+ final HllSketch directSketch;
+ if (curMode == CurMode.HLL) {
+ if (tgtHllType == TgtHllType.HLL_4) {
+ directSketch = new HllSketch(new DirectHll4Array(lgConfigK, srcSeg, true));
+ } else if (tgtHllType == TgtHllType.HLL_6) {
+ directSketch = new HllSketch(new DirectHll6Array(lgConfigK, srcSeg, true));
+ } else { //Hll_8
+ directSketch = new HllSketch(new DirectHll8Array(lgConfigK, srcSeg, true));
+ //rebuild if srcSeg came from a union and was not finalized, rather than throw exception.
+ Union.checkRebuildCurMinNumKxQ(directSketch);
+ }
+ } else if (curMode == CurMode.LIST) {
+ directSketch =
+ new HllSketch(new DirectCouponList(lgConfigK, tgtHllType, curMode, srcSeg, true));
+ } else { //SET
+ directSketch =
+ new HllSketch(new DirectCouponHashSet(lgConfigK, tgtHllType, srcSeg, true));
+ }
+ return directSketch;
+ }
+
+ /**
+ * Return a copy of this sketch onto the Java heap.
+ * @return a copy of this sketch onto the Java heap.
+ */
+ public HllSketch copy() {
+ return new HllSketch(this);
+ }
+
+ /**
+ * Return a deep copy of this sketch onto the Java heap with the specified TgtHllType.
+ * @param tgtHllType the TgtHllType enum
+ * @return a deep copy of this sketch with the specified TgtHllType.
+ */
+ public HllSketch copyAs(final TgtHllType tgtHllType) {
+ return new HllSketch(hllSketchImpl.copyAs(tgtHllType));
+ }
+
+ @Override
+ public double getCompositeEstimate() {
+ return hllSketchImpl.getCompositeEstimate();
+ }
+
+ @Override
+ public double getEstimate() {
+ return hllSketchImpl.getEstimate();
+ }
+
+ double getHipEstimate() {
+ return hllSketchImpl.getHipEstimate();
+ }
+
+ @Override
+ public int getLgConfigK() {
+ return hllSketchImpl.getLgConfigK();
+ }
+
+ @Override
+ public int getCompactSerializationBytes() {
+ return hllSketchImpl.getCompactSerializationBytes();
+ }
+
+ @Override
+ public double getLowerBound(final int numStdDev) {
+ return hllSketchImpl.getLowerBound(numStdDev);
+ }
+
+ /**
+ * Returns the maximum size in bytes that this sketch can grow to given lgConfigK.
+ * However, for the HLL_4 sketch type, this value can be exceeded in extremely rare cases.
+ * If exceeded, it will be larger by only a few percent.
+ *
+ * @param lgConfigK The Log2 of K for the target HLL sketch. This value must be
+ * between 4 and 21 inclusively.
+ * @param tgtHllType the desired Hll type
+ * @return the maximum size in bytes that this sketch can grow to.
+ */
+ public static final int getMaxUpdatableSerializationBytes(final int lgConfigK,
+ final TgtHllType tgtHllType) {
+ final int arrBytes;
+ if (tgtHllType == TgtHllType.HLL_4) {
+ final int auxBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
+ arrBytes = AbstractHllArray.hll4ArrBytes(lgConfigK) + auxBytes;
+ }
+ else if (tgtHllType == TgtHllType.HLL_6) {
+ arrBytes = AbstractHllArray.hll6ArrBytes(lgConfigK);
+ }
+ else { //HLL_8
+ arrBytes = AbstractHllArray.hll8ArrBytes(lgConfigK);
+ }
+ return HLL_BYTE_ARR_START + arrBytes;
+ }
+
+ MemorySegment getMemorySegment() {
+ return hllSketchImpl.getMemorySegment();
+ }
+
+ @Override
+ public TgtHllType getTgtHllType() {
+ return hllSketchImpl.getTgtHllType();
+ }
+
+ @Override
+ public int getUpdatableSerializationBytes() {
+ return hllSketchImpl.getUpdatableSerializationBytes();
+ }
+
+ @Override
+ public double getUpperBound(final int numStdDev) {
+ return hllSketchImpl.getUpperBound(numStdDev);
+ }
+
+ @Override
+ public boolean isCompact() {
+ return hllSketchImpl.isCompact();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return hllSketchImpl.isEmpty();
+ }
+
+ @Override
+ public boolean hasMemorySegment() {
+ return hllSketchImpl.hasMemorySegment();
+ }
+
+ @Override
+ public boolean isOffHeap() {
+ return hllSketchImpl.isOffHeap();
+ }
+
+ @Override
+ boolean isOutOfOrder() {
+ return hllSketchImpl.isOutOfOrder();
+ }
+
+ @Override
+ public boolean isSameResource(final MemorySegment seg) {
+ return hllSketchImpl.isSameResource(seg);
+ }
+
+ void mergeTo(final HllSketch that) {
+ hllSketchImpl.mergeTo(that);
+ }
+
+ HllSketch putOutOfOrderFlag(final boolean oooFlag) {
+ hllSketchImpl.putOutOfOrder(oooFlag);
+ return this;
+ }
+
+ @Override
+ public void reset() {
+ hllSketchImpl = hllSketchImpl.reset();
+ }
+
+ @Override
+ public byte[] toCompactByteArray() {
+ return hllSketchImpl.toCompactByteArray();
+ }
+
+ @Override
+ public byte[] toUpdatableByteArray() {
+ return hllSketchImpl.toUpdatableByteArray();
+ }
+
+ @Override
+ public String toString(final boolean summary, final boolean detail, final boolean auxDetail,
+ final boolean all) {
+ final StringBuilder sb = new StringBuilder();
+ if (summary) {
+ sb.append("### HLL SKETCH SUMMARY: ").append(LS);
+ sb.append(" Log Config K : ").append(getLgConfigK()).append(LS);
+ sb.append(" Hll Target : ").append(getTgtHllType()).append(LS);
+ sb.append(" Current Mode : ").append(getCurMode()).append(LS);
+ sb.append(" MemorySegment : ").append(hasMemorySegment()).append(LS);
+ sb.append(" LB : ").append(getLowerBound(1)).append(LS);
+ sb.append(" Estimate : ").append(getEstimate()).append(LS);
+ sb.append(" UB : ").append(getUpperBound(1)).append(LS);
+ sb.append(" OutOfOrder Flag: ").append(isOutOfOrder()).append(LS);
+ if (getCurMode() == CurMode.HLL) {
+ final AbstractHllArray absHll = (AbstractHllArray) hllSketchImpl;
+ sb.append(" CurMin : ").append(absHll.getCurMin()).append(LS);
+ sb.append(" NumAtCurMin : ").append(absHll.getNumAtCurMin()).append(LS);
+ sb.append(" HipAccum : ").append(absHll.getHipAccum()).append(LS);
+ sb.append(" KxQ0 : ").append(absHll.getKxQ0()).append(LS);
+ sb.append(" KxQ1 : ").append(absHll.getKxQ1()).append(LS);
+ sb.append(" Rebuild KxQ Flg: ").append(absHll.isRebuildCurMinNumKxQFlag()).append(LS);
+ } else {
+ sb.append(" Coupon Count : ")
+ .append(((AbstractCoupons)hllSketchImpl).getCouponCount()).append(LS);
+ }
+ }
+ if (detail) {
+ sb.append("### HLL SKETCH DATA DETAIL: ").append(LS);
+ final PairIterator pitr = iterator();
+ sb.append(pitr.getHeader()).append(LS);
+ if (all) {
+ while (pitr.nextAll()) {
+ sb.append(pitr.getString()).append(LS);
+ }
+ } else {
+ while (pitr.nextValid()) {
+ sb.append(pitr.getString()).append(LS);
+ }
+ }
+ }
+ if (auxDetail && ((getCurMode() == CurMode.HLL) && (getTgtHllType() == TgtHllType.HLL_4))) {
+ final AbstractHllArray absHll = (AbstractHllArray) hllSketchImpl;
+ final PairIterator auxItr = absHll.getAuxIterator();
+ if (auxItr != null) {
+ sb.append("### HLL SKETCH AUX DETAIL: ").append(LS);
+ sb.append(auxItr.getHeader()).append(LS);
+ if (all) {
+ while (auxItr.nextAll()) {
+ sb.append(auxItr.getString()).append(LS);
+ }
+ } else {
+ while (auxItr.nextValid()) {
+ sb.append(auxItr.getString()).append(LS);
+ }
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Returns a human readable string of the preamble of a byte array image of an HllSketch.
+ * @param byteArr the given byte array
+ * @return a human readable string of the preamble of a byte array image of an HllSketch.
+ */
+ public static String toString(final byte[] byteArr) {
+ return PreambleUtil.toString(byteArr);
+ }
+
+ /**
+ * Returns a human readable string of the preamble of a MemorySegment image of an HllSketch.
+ * @param seg the given MemorySegment object
+ * @return a human readable string of the preamble of a MemorySegment image of an HllSketch.
+ */
+ public static String toString(final MemorySegment seg) {
+ return PreambleUtil.toString(seg);
+ }
+
+ //restricted methods
+
+ /**
+ * Returns a PairIterator over the key, value pairs of the HLL array.
+ * @return a PairIterator over the key, value pairs of the HLL array.
+ */
+ PairIterator iterator() {
+ return hllSketchImpl.iterator();
+ }
+
+ @Override
+ CurMode getCurMode() {
+ return hllSketchImpl.getCurMode();
+ }
+
+ @Override
+ void couponUpdate(final int coupon) {
+ if ((coupon >>> KEY_BITS_26 ) == EMPTY) { return; }
+ hllSketchImpl = hllSketchImpl.couponUpdate(coupon);
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllSketchImpl.java b/src/main/java/org/apache/datasketches/hll2/HllSketchImpl.java
new file mode 100644
index 000000000..3393dee65
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllSketchImpl.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import java.lang.foreign.MemorySegment;
+
+/**
+ * The Abstract HllSketch implementation
+ *
+ * @author Lee Rhodes
+ */
+abstract class HllSketchImpl {
+ final int lgConfigK;
+ final TgtHllType tgtHllType;
+ final CurMode curMode;
+
+ HllSketchImpl(final int lgConfigK, final TgtHllType tgtHllType, final CurMode curMode) {
+ this.lgConfigK = lgConfigK;
+ this.tgtHllType = tgtHllType;
+ this.curMode = curMode;
+ }
+
+ /**
+ * Returns a copy of this sketch on the Heap.
+ * The LgConfigK, TgtHllType and CurMode are not changed.
+ * @return Returns a copy of this sketch on the Heap.
+ */
+ abstract HllSketchImpl copy();
+
+ /**
+ * Returns a copy of this sketch on the Heap with the given TgtHllType.
+ * The LgConfigK and CurMode are not changed.
+ * @return Returns a copy of this sketch on the Heap with the given TgtHllType.
+ */
+ abstract HllSketchImpl copyAs(TgtHllType tgtHllType);
+
+ abstract HllSketchImpl couponUpdate(int coupon);
+
+ abstract int getCompactSerializationBytes();
+
+ abstract double getCompositeEstimate();
+
+ CurMode getCurMode() {
+ return curMode;
+ }
+
+ abstract double getEstimate();
+
+ abstract double getHipEstimate();
+
+ int getLgConfigK() {
+ return lgConfigK;
+ }
+
+ abstract double getLowerBound(int numStdDev);
+
+ abstract int getSegDataStart();
+
+ abstract MemorySegment getMemorySegment();
+
+ abstract int getPreInts();
+
+ TgtHllType getTgtHllType() {
+ return tgtHllType;
+ }
+
+ abstract int getUpdatableSerializationBytes();
+
+ abstract double getUpperBound(int numStdDev);
+
+ abstract boolean isCompact();
+
+ abstract boolean isEmpty();
+
+ boolean hasMemorySegment() {
+ return (this instanceof final DirectCouponList dcl && (dcl.seg.scope().isAlive()));
+ }
+
+ abstract boolean isOffHeap();
+
+ abstract boolean isOutOfOrder();
+
+ abstract boolean isRebuildCurMinNumKxQFlag();
+
+ abstract boolean isSameResource(MemorySegment seg);
+
+ abstract PairIterator iterator();
+
+ abstract void mergeTo(HllSketch that);
+
+ abstract void putEmptyFlag(boolean empty);
+
+ abstract void putOutOfOrder(boolean oooFlag);
+
+ abstract void putRebuildCurMinNumKxQFlag(boolean rebuild);
+
+ abstract HllSketchImpl reset();
+
+ abstract byte[] toCompactByteArray();
+
+ abstract byte[] toUpdatableByteArray();
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/HllUtil.java b/src/main/java/org/apache/datasketches/hll2/HllUtil.java
new file mode 100644
index 000000000..a3fc99093
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/HllUtil.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.Math.log;
+import static java.lang.Math.sqrt;
+import static org.apache.datasketches.common.Util.checkBounds;
+import static org.apache.datasketches.hll2.PreambleUtil.HASH_SET_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.LIST_PREINTS;
+import static org.apache.datasketches.hll2.PreambleUtil.extractCurMode;
+import static org.apache.datasketches.hll2.PreambleUtil.extractFamilyId;
+import static org.apache.datasketches.hll2.PreambleUtil.extractPreInts;
+import static org.apache.datasketches.hll2.PreambleUtil.extractSerVer;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.Family;
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesReadOnlyException;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class HllUtil {
+ static final int KEY_BITS_26 = 26;
+ static final int VAL_BITS_6 = 6;
+ static final int KEY_MASK_26 = (1 << KEY_BITS_26) - 1;
+ static final int VAL_MASK_6 = (1 << VAL_BITS_6) - 1;
+ static final int EMPTY = 0;
+ static final int MIN_LOG_K = 4;
+ static final int MAX_LOG_K = 21;
+
+ static final double HLL_HIP_RSE_FACTOR = sqrt(log(2.0)); //.8325546
+ static final double HLL_NON_HIP_RSE_FACTOR = sqrt((3.0 * log(2.0)) - 1.0); //1.03896
+ static final double COUPON_RSE_FACTOR = .409; //at transition point not the asymptote
+
+ static final double COUPON_RSE = COUPON_RSE_FACTOR / (1 << 13);
+
+ static final int LG_INIT_LIST_SIZE = 3;
+ static final int LG_INIT_SET_SIZE = 5;
+ static final int RESIZE_NUMER = 3;
+ static final int RESIZE_DENOM = 4;
+
+ static final int loNibbleMask = 0x0f;
+ static final int hiNibbleMask = 0xf0;
+ static final int AUX_TOKEN = 0xf;
+
+ /**
+ * Log2 table sizes for exceptions based on lgK from 0 to 26.
+ * However, only lgK from 4 to 21 are used.
+ */
+ static final int[] LG_AUX_ARR_INTS = {
+ 0, 2, 2, 2, 2, 2, 2, 3, 3, 3, //0 - 9
+ 4, 4, 5, 5, 6, 7, 8, 9, 10, 11, //10 - 19
+ 12, 13, 14, 15, 16, 17, 18 //20 - 26
+ };
+
+ //Checks
+ static int checkLgK(final int lgK) {
+ if ((lgK >= MIN_LOG_K) && (lgK <= MAX_LOG_K)) { return lgK; }
+ throw new SketchesArgumentException(
+ "Log K must be between 4 and 21, inclusive: " + lgK);
+ }
+
+ static void checkSegSize(final long minBytes, final long capBytes) {
+ if (capBytes < minBytes) {
+ throw new SketchesArgumentException(
+ "Given WritableMemory is not large enough: " + capBytes);
+ }
+ }
+
+ static void checkNumStdDev(final int numStdDev) {
+ if ((numStdDev < 1) || (numStdDev > 3)) {
+ throw new SketchesArgumentException(
+ "NumStdDev may not be less than 1 or greater than 3.");
+ }
+ }
+
+ static CurMode checkPreamble(final MemorySegment seg) {
+ checkBounds(0, 8, seg.byteSize()); //need min 8 bytes
+ final int preInts = extractPreInts(seg);
+ checkBounds(0, (long)preInts * Integer.BYTES, seg.byteSize());
+ final int serVer = extractSerVer(seg);
+ final int famId = extractFamilyId(seg);
+ final CurMode curMode = extractCurMode(seg);
+ if (
+ (famId != Family.HLL.getID())
+ || (serVer != 1)
+ || ((preInts != LIST_PREINTS) && (preInts != HASH_SET_PREINTS) && (preInts != HLL_PREINTS))
+ || ((curMode == CurMode.LIST) && (preInts != LIST_PREINTS))
+ || ((curMode == CurMode.SET) && (preInts != HASH_SET_PREINTS))
+ || ((curMode == CurMode.HLL) && (preInts != HLL_PREINTS))
+ ) {
+ throw new SketchesArgumentException("Possible Corruption, Invalid Preamble:"
+ + PreambleUtil.toString(seg));
+ }
+ return curMode;
+ }
+
+ //Exceptions
+ static void noWriteAccess() {
+ throw new SketchesReadOnlyException(
+ "This sketch is compact or does not have write access to the underlying resource.");
+ }
+
+ //Used for thrown exceptions
+ static String pairString(final int pair) {
+ return "SlotNo: " + getPairLow26(pair) + ", Value: "
+ + getPairValue(pair);
+ }
+
+ //Pairs
+ static int pair(final int slotNo, final int value) {
+ return (value << KEY_BITS_26) | (slotNo & KEY_MASK_26);
+ }
+
+ static int getPairLow26(final int coupon) {
+ return coupon & KEY_MASK_26;
+ }
+
+ static int getPairValue(final int coupon) {
+ return coupon >>> KEY_BITS_26;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/IntArrayPairIterator.java b/src/main/java/org/apache/datasketches/hll2/IntArrayPairIterator.java
new file mode 100644
index 000000000..0a40a8754
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/IntArrayPairIterator.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+
+/**
+ * Iterates over an on-heap integer array of pairs extracting
+ * the components of the pair at a given index.
+ *
+ * @author Lee Rhodes
+ */
+class IntArrayPairIterator extends PairIterator {
+ private final int[] array;
+ private final int arrLen;
+ private final int slotMask;
+ private int index;
+ private int pair;
+
+ //used by CouponList, HeapAuxHashMap
+ IntArrayPairIterator(final int[] array, final int lgConfigK) {
+ this.array = array;
+ slotMask = (1 << lgConfigK) - 1;
+ arrLen = array.length;
+ index = - 1;
+ }
+
+ @Override
+ public int getIndex() {
+ return index;
+ }
+
+ @Override
+ public int getKey() {
+ return HllUtil.getPairLow26(pair);
+ }
+
+ @Override
+ public int getPair() {
+ return pair;
+ }
+
+ @Override
+ public int getSlot() {
+ return getKey() & slotMask;
+ }
+
+ @Override
+ public int getValue() {
+ return HllUtil.getPairValue(pair);
+ }
+
+ @Override
+ public boolean nextAll() {
+ if (++index < arrLen) {
+ pair = array[index];
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean nextValid() {
+ while (++index < arrLen) {
+ final int pair = array[index];
+ if (pair != EMPTY) {
+ this.pair = pair;
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/IntMemorySegmentPairIterator.java b/src/main/java/org/apache/datasketches/hll2/IntMemorySegmentPairIterator.java
new file mode 100644
index 000000000..b1be847e8
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/IntMemorySegmentPairIterator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+
+import java.lang.foreign.MemorySegment;
+
+/**
+ * Iterates within a given Memory extracting integer pairs.
+ *
+ * @author Lee Rhodes
+ */
+class IntMemorySegmentPairIterator extends PairIterator {
+ private final MemorySegment seg;
+ private final long offsetBytes;
+ private final int arrLen;
+ private final int slotMask;
+ private int index;
+ private int pair;
+
+ //Used by DirectAuxHashMap, DirectCouponList
+ IntMemorySegmentPairIterator(
+ final MemorySegment seg, final long offsetBytes, final int arrayLength, final int lgConfigK) {
+ this.seg = seg;
+ this.offsetBytes = offsetBytes;
+ this.arrLen = arrayLength;
+ slotMask = (1 << lgConfigK) - 1;
+ index = -1;
+ }
+
+ IntMemorySegmentPairIterator(final byte[] byteArr, final long offsetBytes, final int lengthPairs,
+ final int lgConfigK) {
+ this(MemorySegment.ofArray(byteArr), offsetBytes, lengthPairs, lgConfigK);
+ }
+
+ @Override
+ public int getIndex() {
+ return index;
+ }
+
+ @Override
+ public int getKey() {
+ return HllUtil.getPairLow26(pair);
+ }
+
+ @Override
+ public int getPair() {
+ return pair;
+ }
+
+ @Override
+ public int getSlot() {
+ return getKey() & slotMask;
+ }
+
+ @Override
+ public int getValue() {
+ return HllUtil.getPairValue(pair);
+ }
+
+ @Override
+ public boolean nextAll() {
+ if (++index < arrLen) {
+ pair = pair();
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean nextValid() {
+ while (++index < arrLen) {
+ final int pair = pair();
+ if (pair != EMPTY) {
+ this.pair = pair;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ int pair() {
+ return seg.get(JAVA_INT_UNALIGNED, offsetBytes + (index << 2));
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/PairIterator.java b/src/main/java/org/apache/datasketches/hll2/PairIterator.java
new file mode 100644
index 000000000..6bed27e13
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/PairIterator.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * @author Lee Rhodes
+ */
+abstract class PairIterator {
+
+ /**
+ * Gets the header string for a list of pairs
+ * @return the header string for a list of pairs
+ */
+ String getHeader() {
+ return String.format("%10s%10s%10s%6s", "Index", "Key", "Slot", "Value");
+ }
+
+ /**
+ * In LIST and SET modes, this gets the iterating index into the integer array of HLL key/value
+ * pairs.
+ * In HLL mode, this is the iterating index into the hypothetical array of HLL values, which may
+ * be physically contructed differently based on the compaction scheme (HLL_4, HLL_6, HLL_8).
+ * @return the index.
+ */
+ abstract int getIndex();
+
+ /**
+ * Gets the key, the low 26 bits of an pair, and can be up to 26 bits in length.
+ * @return the key
+ */
+ abstract int getKey();
+
+ /**
+ * Gets the key, value pair as a single int where the key is the lower 26 bits
+ * and the value is in the upper 6 bits.
+ * @return the key, value pair.
+ */
+ abstract int getPair();
+
+ /**
+ * Gets the target or actual HLL slot number, which is derived from the key and LgConfigK.
+ * The slot number is the index into a hypothetical array of length K and has LgConfigK bits.
+ * If in LIST or SET mode this is the index into the hypothetical target HLL array of size K.
+ * In HLL mode, this will be the actual index into the hypothetical target HLL array of size K.
+ * @return the target or actual HLL slot number.
+ */
+ abstract int getSlot();
+
+ /**
+ * Gets the current pair as a string
+ * @return the current pair as a string
+ */
+ String getString() {
+ final int index = getIndex();
+ final int key = getKey();
+ final int slot = getSlot();
+ final int value = getValue();
+ return String.format("%10d%10d%10d%6d", index, key, slot, value);
+ }
+
+ /**
+ * Gets the HLL value of a particular slot or pair.
+ * @return the HLL value of a particular slot or pair.
+ */
+ abstract int getValue();
+
+ /**
+ * Returns true at the next pair in sequence.
+ * If false, the iteration is done.
+ * @return true at the next pair in sequence.
+ */
+ abstract boolean nextAll();
+
+ /**
+ * Returns true at the next pair where getKey() and getValue() are valid.
+ * If false, the iteration is done.
+ * @return true at the next pair where getKey() and getValue() are valid.
+ */
+ abstract boolean nextValid();
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/PreambleUtil.java b/src/main/java/org/apache/datasketches/hll2/PreambleUtil.java
new file mode 100644
index 000000000..bfef754da
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/PreambleUtil.java
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED;
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.common.Util.LS;
+import static org.apache.datasketches.common.Util.ceilingPowerOf2;
+import static org.apache.datasketches.common.Util.exactLog2OfLong;
+import static org.apache.datasketches.common.Util.zeroPad;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.HllUtil.LG_INIT_SET_SIZE;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_DENOM;
+import static org.apache.datasketches.hll2.HllUtil.RESIZE_NUMER;
+
+import java.lang.foreign.MemorySegment;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.common.Family;
+
+//@formatter:off
+/**
+ * <pre>
+ * CouponList Layout
+ * Long || Start Byte Adr, Big Endian Illustration
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 || Mode | ListCnt| Flags | LgArr | lgK | FamID | SerVer | PI=2 |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 || |------Coupon Int List Start--------|
+ * </pre>
+ *
+ * <pre>
+ * CouponHashSet Layout
+ * Long || Start Byte Adr, Big Endian Illustration
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 || Mode | | Flags | LgArr | lgK | FamID | SerVer | PI=3 |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 ||-----Coupon Int Hash Set Start-----|---------Hash Set Count------------|
+ * </pre>
+ *
+ * <pre>
+ * HllArray Layout
+ * Long || Start Byte Adr, Big Endian Illustration
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 || Mode | CurMin | Flags | LgArr | lgK | FamID | SerVer | PI=10 |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 ||-------------------------------HIP Accum-------------------------------|
+ *
+ * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
+ * 2 ||----------------------------------KxQ0---------------------------------|
+ *
+ * || 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 |
+ * 3 ||----------------------------------KxQ1---------------------------------|
+ *
+ * || 39 | 38 | 37 | 36 | 35 | 34 | 33 | 32 |
+ * 4 ||-------------Aux Count-------------|----------Num At Cur Min-----------|
+ *
+ * || 47 | 46 | 45 | 44 | 43 | 42 | 41 | 40 |
+ * 5 ||...................................|------Start of HLL_X Byte Array----|
+ *
+ * N ||----End of Byte Array for HLL_4----|...................................|
+ * N+1 ||...................................|-----Start of Aux Array for HLL_4--|
+ * </pre>
+ * If in compact form exceptions array will be compacted.
+ *
+ * @author Lee Rhodes
+ */
+final class PreambleUtil {
+
+ private PreambleUtil() {}
+
+ // ###### DO NOT MESS WITH THIS ...
+ // Preamble byte start addresses
+ // First 8 Bytes:
+ static int PREAMBLE_INTS_BYTE = 0;
+ static int SER_VER_BYTE = 1;
+ static int FAMILY_BYTE = 2;
+ static int LG_K_BYTE = 3;
+ static int LG_ARR_BYTE = 4; //used for LIST, SET & HLL_4
+ static int FLAGS_BYTE = 5;
+ static int LIST_COUNT_BYTE = 6;
+ static int HLL_CUR_MIN_BYTE = 6;
+ static int MODE_BYTE = 7; //lo2bits = curMode, next 2 bits = tgtHllType
+ //mode encoding of combined CurMode and TgtHllType:
+ // Dec Lo4Bits TgtHllType, CurMode
+ // 0 0000 HLL_4, LIST
+ // 1 0001 HLL_4, SET
+ // 2 0010 HLL_4, HLL
+ // 4 0100 HLL_6, LIST
+ // 5 0101 HLL_6, SET
+ // 6 0110 HLL_6, HLL
+ // 8 1000 HLL_8, LIST
+ // 9 1001 HLL_8, SET
+ // 10 1010 HLL_8, HLL
+
+ //Coupon List
+ static int LIST_INT_ARR_START = 8;
+
+ //Coupon Hash Set
+ static int HASH_SET_COUNT_INT = 8;
+ static int HASH_SET_INT_ARR_START = 12;
+
+ //HLL
+ static int HIP_ACCUM_DOUBLE = 8;
+ static int KXQ0_DOUBLE = 16;
+ static int KXQ1_DOUBLE = 24;
+ static int CUR_MIN_COUNT_INT = 32;
+ static int AUX_COUNT_INT = 36;
+ static int HLL_BYTE_ARR_START = 40;
+
+ //Flag bit masks
+ static final int BIG_ENDIAN_FLAG_MASK = 1; //Set but not read. Reserved.
+ static final int READ_ONLY_FLAG_MASK = 2; //Set but not read. Reserved.
+ static final int EMPTY_FLAG_MASK = 4;
+ static final int COMPACT_FLAG_MASK = 8;
+ static final int OUT_OF_ORDER_FLAG_MASK = 16;
+ static final int REBUILD_CURMIN_NUM_KXQ_MASK = 32; //used only by Union
+
+ //Mode byte masks
+ static final int CUR_MODE_MASK = 3;
+ static final int TGT_HLL_TYPE_MASK = 12;
+
+ //Other constants
+ static final int SER_VER = 1;
+ static final int FAMILY_ID = 7;
+ static final int LIST_PREINTS = 2;
+ static final int HASH_SET_PREINTS = 3;
+ static final int HLL_PREINTS = 10;
+ static final boolean NATIVE_ORDER_IS_BIG_ENDIAN =
+ (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN);
+
+ static String toString(final byte[] byteArr) {
+ final MemorySegment seg = MemorySegment.ofArray(byteArr);
+ return toString(seg);
+ }
+
+ static String toString(final MemorySegment seg) {
+ //First 8 bytes
+ final int preInts = seg.get(JAVA_BYTE, PREAMBLE_INTS_BYTE);
+ final int serVer = seg.get(JAVA_BYTE, SER_VER_BYTE);
+ final Family family = Family.idToFamily(seg.get(JAVA_BYTE, FAMILY_BYTE));
+ final int lgK = seg.get(JAVA_BYTE, LG_K_BYTE);
+ final int lgArr = seg.get(JAVA_BYTE, LG_ARR_BYTE);
+ final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE);
+ //Flags
+ final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags);
+ final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0;
+ final String nativeOrder = ByteOrder.nativeOrder().toString();
+ final boolean compact = (flags & COMPACT_FLAG_MASK) > 0;
+ final boolean oooFlag = (flags & OUT_OF_ORDER_FLAG_MASK) > 0;
+ final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0;
+ final boolean empty = (flags & EMPTY_FLAG_MASK) > 0;
+ final boolean rebuildKxQ = (flags & REBUILD_CURMIN_NUM_KXQ_MASK) > 0;
+
+ final int hllCurMin = seg.get(JAVA_BYTE, HLL_CUR_MIN_BYTE);
+ final int listCount = hllCurMin;
+ final int modeByte = seg.get(JAVA_BYTE, MODE_BYTE);
+ final CurMode curMode = CurMode.fromOrdinal(modeByte & 3);
+ final TgtHllType tgtHllType = TgtHllType.fromOrdinal((modeByte >>> 2) & 3);
+
+ double hipAccum = 0;
+ double kxq0 = 0;
+ double kxq1 = 0;
+ int hashSetCount = 0;
+ int curMinCount = 0;
+ int exceptionCount = 0;
+
+ if (curMode == CurMode.SET) {
+ hashSetCount = seg.get(JAVA_INT_UNALIGNED, HASH_SET_COUNT_INT);
+ }
+ else if (curMode == CurMode.HLL) {
+ hipAccum = seg.get(JAVA_DOUBLE_UNALIGNED, HIP_ACCUM_DOUBLE);
+ kxq0 = seg.get(JAVA_DOUBLE_UNALIGNED, KXQ0_DOUBLE);
+ kxq1 = seg.get(JAVA_DOUBLE_UNALIGNED, KXQ1_DOUBLE);
+ curMinCount = seg.get(JAVA_INT_UNALIGNED, CUR_MIN_COUNT_INT);
+ exceptionCount = seg.get(JAVA_INT_UNALIGNED, AUX_COUNT_INT);
+ }
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(LS);
+ sb.append("### HLL SKETCH PREAMBLE:").append(LS);
+ sb.append("Byte 0: Preamble Ints : ").append(preInts).append(LS);
+ sb.append("Byte 1: SerVer : ").append(serVer).append(LS);
+ sb.append("Byte 2: Family : ").append(family).append(LS);
+ sb.append("Byte 3: lgK : ").append(lgK).append(LS);
+ //expand byte 4: LgArr
+ if (curMode == CurMode.LIST) {
+ sb.append("Byte 4: LgArr: List Arr : ").append(lgArr).append(LS);
+ }
+ if (curMode == CurMode.SET) {
+ sb.append("Byte 4: LgArr: Hash Set Arr : ").append(lgArr).append(LS);
+ }
+ if (curMode == CurMode.HLL) {
+ sb.append("Byte 4: LgArr or Aux LgArr : ").append(lgArr).append(LS);
+ }
+ //expand byte 5: Flags
+ sb.append("Byte 5: Flags: : ").append(flagsStr).append(LS);
+ sb.append(" BIG_ENDIAN_STORAGE : ").append(bigEndian).append(LS);
+ sb.append(" (Native Byte Order) : ").append(nativeOrder).append(LS);
+ sb.append(" READ_ONLY : ").append(readOnly).append(LS);
+ sb.append(" EMPTY : ").append(empty).append(LS);
+ sb.append(" COMPACT : ").append(compact).append(LS);
+ sb.append(" OUT_OF_ORDER : ").append(oooFlag).append(LS);
+ sb.append(" REBUILD_KXQ : ").append(rebuildKxQ).append(LS);
+ //expand byte 6: ListCount, CurMin
+ if (curMode == CurMode.LIST) {
+ sb.append("Byte 6: List Count/CurMin : ").append(listCount).append(LS);
+ }
+ if (curMode == CurMode.SET) {
+ sb.append("Byte 6: (not used) : ").append(LS);
+ }
+ if (curMode == CurMode.HLL) {
+ sb.append("Byte 6: Cur Min : ").append(hllCurMin).append(LS);
+ }
+ final String modes = curMode.toString() + ", " + tgtHllType.toString();
+ sb.append("Byte 7: Mode : ").append(modes).append(LS);
+ if (curMode == CurMode.SET) {
+ sb.append("Hash Set Count : ").append(hashSetCount).append(LS);
+ }
+ if (curMode == CurMode.HLL) {
+ sb.append("HIP Accum : ").append(hipAccum).append(LS);
+ sb.append("KxQ0 : ").append(kxq0).append(LS);
+ sb.append("KxQ1 : ").append(kxq1).append(LS);
+ sb.append("Num At Cur Min : ").append(curMinCount).append(LS);
+ sb.append("Aux Count : ").append(exceptionCount).append(LS);
+ }
+ sb.append("### END HLL SKETCH PREAMBLE").append(LS);
+ return sb.toString();
+ }
+ //@formatter:on
+
+ static int extractPreInts(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, PREAMBLE_INTS_BYTE) & 0X3F;
+ }
+
+ static void insertPreInts(final MemorySegment wseg, final int preInts) {
+ wseg.set(JAVA_BYTE, PREAMBLE_INTS_BYTE, (byte) (preInts & 0X3F));
+ }
+
+ static int extractSerVer(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, SER_VER_BYTE) & 0XFF;
+ }
+
+ static void insertSerVer(final MemorySegment wseg) {
+ wseg.set(JAVA_BYTE, SER_VER_BYTE, (byte) SER_VER);
+ }
+
+ static int extractFamilyId(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF;
+ }
+
+ static void insertFamilyId(final MemorySegment wseg) {
+ wseg.set(JAVA_BYTE, FAMILY_BYTE, (byte) FAMILY_ID);
+ }
+
+ static int extractLgK(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, LG_K_BYTE) & 0XFF;
+ }
+
+ static void insertLgK(final MemorySegment wseg, final int lgK) {
+ wseg.set(JAVA_BYTE, LG_K_BYTE, (byte) lgK);
+ }
+
+ static int extractLgArr(final MemorySegment seg) {
+ final int lgArr = seg.get(JAVA_BYTE, LG_ARR_BYTE) & 0XFF;
+ return lgArr;
+ }
+
+ static void insertLgArr(final MemorySegment wseg, final int lgArr) {
+ wseg.set(JAVA_BYTE, LG_ARR_BYTE, (byte) lgArr);
+ }
+
+ static int extractListCount(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, LIST_COUNT_BYTE) & 0XFF;
+ }
+
+ static void insertListCount(final MemorySegment wseg, final int listCnt) {
+ wseg.set(JAVA_BYTE, LIST_COUNT_BYTE, (byte) listCnt);
+ }
+
+ static int extractCurMin(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, HLL_CUR_MIN_BYTE) & 0XFF;
+ }
+
+ static void insertCurMin(final MemorySegment wseg, final int curMin) {
+ wseg.set(JAVA_BYTE, HLL_CUR_MIN_BYTE, (byte) curMin);
+ }
+
+ static double extractHipAccum(final MemorySegment seg) {
+ return seg.get(JAVA_DOUBLE_UNALIGNED, HIP_ACCUM_DOUBLE);
+ }
+
+ static void insertHipAccum(final MemorySegment wseg, final double hipAccum) {
+ wseg.set(JAVA_DOUBLE_UNALIGNED, HIP_ACCUM_DOUBLE, hipAccum);
+ }
+
+ static double extractKxQ0(final MemorySegment seg) {
+ return seg.get(JAVA_DOUBLE_UNALIGNED, KXQ0_DOUBLE);
+ }
+
+ static void insertKxQ0(final MemorySegment wseg, final double kxq0) {
+ wseg.set(JAVA_DOUBLE_UNALIGNED, KXQ0_DOUBLE, kxq0);
+ }
+
+ static double extractKxQ1(final MemorySegment seg) {
+ return seg.get(JAVA_DOUBLE_UNALIGNED, KXQ1_DOUBLE);
+ }
+
+ static void insertKxQ1(final MemorySegment wseg, final double kxq1) {
+ wseg.set(JAVA_DOUBLE_UNALIGNED, KXQ1_DOUBLE, kxq1);
+ }
+
+ static int extractHashSetCount(final MemorySegment seg) {
+ return seg.get(JAVA_INT_UNALIGNED, HASH_SET_COUNT_INT);
+ }
+
+ static void insertHashSetCount(final MemorySegment wseg, final int hashSetCnt) {
+ wseg.set(JAVA_INT_UNALIGNED, HASH_SET_COUNT_INT, hashSetCnt);
+ }
+
+ static int extractNumAtCurMin(final MemorySegment seg) {
+ return seg.get(JAVA_INT_UNALIGNED, CUR_MIN_COUNT_INT);
+ }
+
+ static void insertNumAtCurMin(final MemorySegment wseg, final int numAtCurMin) {
+ wseg.set(JAVA_INT_UNALIGNED, CUR_MIN_COUNT_INT, numAtCurMin);
+ }
+
+ static int extractAuxCount(final MemorySegment seg) {
+ return seg.get(JAVA_INT_UNALIGNED, AUX_COUNT_INT);
+ }
+
+ static void insertAuxCount(final MemorySegment wseg, final int auxCount) {
+ wseg.set(JAVA_INT_UNALIGNED, AUX_COUNT_INT, auxCount);
+ }
+
+ //Mode bits
+ static void insertCurMode(final MemorySegment wseg, final CurMode curMode) {
+ final int curModeId = curMode.ordinal();
+ int mode = wseg.get(JAVA_BYTE, MODE_BYTE) & ~CUR_MODE_MASK; //strip bits 0, 1
+ mode |= (curModeId & CUR_MODE_MASK);
+ wseg.set(JAVA_BYTE, MODE_BYTE, (byte) mode);
+ }
+
+ static CurMode extractCurMode(final MemorySegment seg) {
+ final int curModeId = seg.get(JAVA_BYTE, MODE_BYTE) & CUR_MODE_MASK;
+ return CurMode.fromOrdinal(curModeId);
+ }
+
+ static void insertTgtHllType(final MemorySegment wseg, final TgtHllType tgtHllType) {
+ final int typeId = tgtHllType.ordinal();
+ int mode = wseg.get(JAVA_BYTE, MODE_BYTE) & ~TGT_HLL_TYPE_MASK; //strip bits 2, 3
+ mode |= (typeId << 2) & TGT_HLL_TYPE_MASK;
+ wseg.set(JAVA_BYTE, MODE_BYTE, (byte) mode);
+ }
+
+ static TgtHllType extractTgtHllType(final MemorySegment seg) {
+ final int typeId = seg.get(JAVA_BYTE, MODE_BYTE) & TGT_HLL_TYPE_MASK;
+ return TgtHllType.fromOrdinal(typeId >>> 2);
+ }
+
+ static void insertModes(final MemorySegment wseg, final TgtHllType tgtHllType,
+ final CurMode curMode) {
+ final int curModeId = curMode.ordinal() & 3;
+ final int typeId = (tgtHllType.ordinal() & 3) << 2;
+ final int mode = typeId | curModeId;
+ wseg.set(JAVA_BYTE, MODE_BYTE, (byte) mode);
+ }
+
+ //Flags
+ static void insertEmptyFlag(final MemorySegment wseg, final boolean empty) {
+ int flags = wseg.get(JAVA_BYTE, FLAGS_BYTE);
+ if (empty) { flags |= EMPTY_FLAG_MASK; }
+ else { flags &= ~EMPTY_FLAG_MASK; }
+ wseg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags);
+ }
+
+ static boolean extractEmptyFlag(final MemorySegment seg) {
+ final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE);
+ return (flags & EMPTY_FLAG_MASK) > 0;
+ }
+
+ static void insertCompactFlag(final MemorySegment wseg, final boolean compact) {
+ int flags = wseg.get(JAVA_BYTE, FLAGS_BYTE);
+ if (compact) { flags |= COMPACT_FLAG_MASK; }
+ else { flags &= ~COMPACT_FLAG_MASK; }
+ wseg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags);
+ }
+
+ static boolean extractCompactFlag(final MemorySegment seg) {
+ final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE);
+ return (flags & COMPACT_FLAG_MASK) > 0;
+ }
+
+ static void insertOooFlag(final MemorySegment wseg, final boolean oooFlag) {
+ int flags = wseg.get(JAVA_BYTE, FLAGS_BYTE);
+ if (oooFlag) { flags |= OUT_OF_ORDER_FLAG_MASK; }
+ else { flags &= ~OUT_OF_ORDER_FLAG_MASK; }
+ wseg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags);
+ }
+
+ static boolean extractOooFlag(final MemorySegment seg) {
+ final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE);
+ return (flags & OUT_OF_ORDER_FLAG_MASK) > 0;
+ }
+
+ static void insertRebuildCurMinNumKxQFlag(final MemorySegment wseg, final boolean rebuild) {
+ int flags = wseg.get(JAVA_BYTE, FLAGS_BYTE);
+ if (rebuild) { flags |= REBUILD_CURMIN_NUM_KXQ_MASK; }
+ else { flags &= ~REBUILD_CURMIN_NUM_KXQ_MASK; }
+ wseg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags);
+ }
+
+ static boolean extractRebuildCurMinNumKxQFlag(final MemorySegment seg) {
+ final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE);
+ return (flags & REBUILD_CURMIN_NUM_KXQ_MASK) > 0;
+ }
+
+ static void insertFlags(final MemorySegment wseg, final int flags) {
+ wseg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags);
+ }
+
+ static int extractFlags(final MemorySegment seg) {
+ return seg.get(JAVA_BYTE, FLAGS_BYTE) & 0XFF;
+ }
+
+ //Other
+ static int extractInt(final MemorySegment seg, final long byteOffset) {
+ return seg.get(JAVA_INT_UNALIGNED, byteOffset);
+ }
+
+ static void insertInt(final MemorySegment wseg, final long byteOffset, final int value) {
+ wseg.set(JAVA_INT_UNALIGNED, byteOffset, value);
+ }
+
+ static int computeLgArr(final MemorySegment seg, final int count, final int lgConfigK) {
+ //value is missing, recompute
+ final CurMode curMode = extractCurMode(seg);
+ if (curMode == CurMode.LIST) { return HllUtil.LG_INIT_LIST_SIZE; }
+ int ceilPwr2 = ceilingPowerOf2(count);
+ if ((RESIZE_DENOM * count) > (RESIZE_NUMER * ceilPwr2)) { ceilPwr2 <<= 1; }
+ if (curMode == CurMode.SET) {
+ return Math.max(LG_INIT_SET_SIZE, exactLog2OfLong(ceilPwr2));
+ }
+ //only used for HLL4
+ return Math.max(LG_AUX_ARR_INTS[lgConfigK], exactLog2OfLong(ceilPwr2));
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/RelativeErrorTables.java b/src/main/java/org/apache/datasketches/hll2/RelativeErrorTables.java
new file mode 100644
index 000000000..31b6cb58c
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/RelativeErrorTables.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+final class RelativeErrorTables {
+
+ /**
+ * Return Relative Error for UB or LB for HIP or Non-HIP as a function of numStdDev.
+ * @param upperBound true if for upper bound
+ * @param oooFlag true if for Non-HIP
+ * @param lgK must be between 4 and 12 inclusive
+ * @param stdDev must be between 1 and 3 inclusive
+ * @return Relative Error for UB or LB for HIP or Non-HIP as a function of numStdDev.
+ */
+ static double getRelErr(final boolean upperBound, final boolean oooFlag,
+ final int lgK, final int stdDev) {
+ final int idx = ((lgK - 4) * 3) + (stdDev - 1);
+ final int sw = (oooFlag ? 2 : 0) | (upperBound ? 1 : 0);
+ double f = 0;
+ switch (sw) {
+ case 0 : { //HIP, LB
+ f = HIP_LB[idx];
+ break;
+ }
+ case 1 : { //HIP, UB
+ f = HIP_UB[idx];
+ break;
+ }
+ case 2 : { //NON_HIP, LB
+ f = NON_HIP_LB[idx];
+ break;
+ }
+ case 3 : { //NON_HIP, UB
+ f = NON_HIP_UB[idx];
+ break;
+ }
+ default: f = 0;//can't happen
+ }
+ return f;
+ }
+
+ //case 0
+ private static double[] HIP_LB = //sd 1, 2, 3
+ { //Q(.84134), Q(.97725), Q(.99865) respectively
+ 0.207316195, 0.502865572, 0.882303765, //4
+ 0.146981579, 0.335426881, 0.557052, //5
+ 0.104026721, 0.227683872, 0.365888317, //6
+ 0.073614601, 0.156781585, 0.245740374, //7
+ 0.05205248, 0.108783763, 0.168030442, //8
+ 0.036770852, 0.075727545, 0.11593785, //9
+ 0.025990219, 0.053145536, 0.080772263, //10
+ 0.018373987, 0.037266176, 0.056271814, //11
+ 0.012936253, 0.02613829, 0.039387631, //12
+ };
+
+ //case 1
+ private static double[] HIP_UB = //sd 1, 2, 3
+ { //Q(.15866), Q(.02275), Q(.00135) respectively
+ -0.207805347, -0.355574279, -0.475535095, //4
+ -0.146988328, -0.262390832, -0.360864026, //5
+ -0.103877775, -0.191503663, -0.269311582, //6
+ -0.073452978, -0.138513438, -0.198487447, //7
+ -0.051982806, -0.099703123, -0.144128618, //8
+ -0.036768609, -0.07138158, -0.104430324, //9
+ -0.025991325, -0.050854296, -0.0748143, //10
+ -0.01834533, -0.036121138, -0.05327616, //11
+ -0.012920332, -0.025572893, -0.037896952, //12
+ };
+
+ //case 2
+ private static double[] NON_HIP_LB = //sd 1, 2, 3
+ { //Q(.84134), Q(.97725), Q(.99865) respectively
+ 0.254409839, 0.682266712, 1.304022158, //4
+ 0.181817353, 0.443389054, 0.778776219, //5
+ 0.129432281, 0.295782195, 0.49252279, //6
+ 0.091640655, 0.201175925, 0.323664385, //7
+ 0.064858051, 0.138523393, 0.218805328, //8
+ 0.045851855, 0.095925072, 0.148635751, //9
+ 0.032454144, 0.067009668, 0.102660669, //10
+ 0.022921382, 0.046868565, 0.071307398, //11
+ 0.016155679, 0.032825719, 0.049677541 //12
+ };
+
+ //case 3
+ private static double[] NON_HIP_UB = //sd 1, 2, 3
+ { //Q(.15866), Q(.02275), Q(.00135) respectively
+ -0.256980172, -0.411905944, -0.52651057, //4
+ -0.182332109, -0.310275547, -0.412660505, //5
+ -0.129314228, -0.230142294, -0.315636197, //6
+ -0.091584836, -0.16834013, -0.236346847, //7
+ -0.06487411, -0.122045231, -0.174112107, //8
+ -0.04591465, -0.08784505, -0.126917615, //9
+ -0.032433119, -0.062897613, -0.091862929, //10
+ -0.022960633, -0.044875401, -0.065736049, //11
+ -0.016186662, -0.031827816, -0.046973459 //12
+ };
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/TgtHllType.java b/src/main/java/org/apache/datasketches/hll2/TgtHllType.java
new file mode 100644
index 000000000..422f851cc
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/TgtHllType.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+/**
+ * Specifies the target type of HLL sketch to be created. It is a target in that the actual
+ * allocation of the HLL array is deferred until sufficient number of items have been received by
+ * the warm-up phases.
+ *
+ * <p>These three target types are isomorphic representations of the same underlying HLL algorithm.
+ * Thus, given the same value of <i>lgConfigK</i> and the same input, all three HLL target types
+ * will produce identical estimates and have identical error distributions.</p>
+ *
+ * <p>The memory usage and the serialization size of the sketch during this early warmup phase starts
+ * out very small (8 bytes, when empty) and then grows in increments of 4 bytes as required
+ * until the full HLL array is allocated. This transition point occurs at about 10% of K for
+ * sketches where lgConfigK is > 8.</p>
+ *
+ * <ul>
+ * <li><b>HLL 8</b> This uses an 8-bit byte per HLL bucket. It is generally the
+ * fastest in terms of update time, but has the largest storage footprint of about
+ * <i>K</i> bytes.</li>
+ *
+ * <li><b>HLL 6</b> This uses a 6-bit field per HLL bucket. It is the generally the next fastest
+ * in terms of update time with a storage footprint of about <i>3/4 * K</i> bytes.</li>
+ *
+ * <li><b>HLL 4</b> This uses a 4-bit field per HLL bucket and for large counts may require
+ * the use of a small internal auxiliary array for storing statistical exceptions, which are rare.
+ * For the values of <i>lgConfigK > 13</i> (<i>K</i> = 8192),
+ * this additional array adds about 3% to the overall storage. It is generally the slowest in
+ * terms of update time, but has the smallest storage footprint of about
+ * <i>K/2 * 1.03</i> bytes.</li>
+ * </ul>
+ * @author Lee Rhodes
+ */
+public enum TgtHllType {
+ /**
+ * An HLL sketch with a bin size of 4 bits
+ */
+ HLL_4,
+ /**
+ * An HLL sketch with a bin size of 6 bits
+ */
+ HLL_6,
+ /**
+ * An Hll Sketch with a bin size of 8 bits
+ */
+ HLL_8;
+
+ private static final TgtHllType values[] = values();
+
+ /**
+ * Convert the typeId to the enum type
+ * @param typeId the given typeId
+ * @return the enum type
+ */
+ public static final TgtHllType fromOrdinal(final int typeId) {
+ return values[typeId];
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/ToByteArrayImpl.java b/src/main/java/org/apache/datasketches/hll2/ToByteArrayImpl.java
new file mode 100644
index 000000000..26138e0df
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/ToByteArrayImpl.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED;
+import static org.apache.datasketches.hll2.AbstractCoupons.find;
+import static org.apache.datasketches.hll2.HllUtil.LG_AUX_ARR_INTS;
+import static org.apache.datasketches.hll2.PreambleUtil.AUX_COUNT_INT;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.insertAuxCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCompactFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertCurMode;
+import static org.apache.datasketches.hll2.PreambleUtil.insertEmptyFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertFamilyId;
+import static org.apache.datasketches.hll2.PreambleUtil.insertHashSetCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertHipAccum;
+import static org.apache.datasketches.hll2.PreambleUtil.insertInt;
+import static org.apache.datasketches.hll2.PreambleUtil.insertKxQ0;
+import static org.apache.datasketches.hll2.PreambleUtil.insertKxQ1;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgArr;
+import static org.apache.datasketches.hll2.PreambleUtil.insertLgK;
+import static org.apache.datasketches.hll2.PreambleUtil.insertListCount;
+import static org.apache.datasketches.hll2.PreambleUtil.insertNumAtCurMin;
+import static org.apache.datasketches.hll2.PreambleUtil.insertOooFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertPreInts;
+import static org.apache.datasketches.hll2.PreambleUtil.insertRebuildCurMinNumKxQFlag;
+import static org.apache.datasketches.hll2.PreambleUtil.insertSerVer;
+import static org.apache.datasketches.hll2.PreambleUtil.insertTgtHllType;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesStateException;
+
+/**
+ * @author Lee Rhodes
+ */
+class ToByteArrayImpl {
+
+ // To byte array used by the heap HLL modes.
+ static final byte[] toHllByteArray(final AbstractHllArray impl, final boolean compact) {
+ int auxBytes = 0;
+ if (impl.tgtHllType == TgtHllType.HLL_4) {
+ final AuxHashMap auxHashMap = impl.getAuxHashMap();
+ if (auxHashMap != null) {
+ auxBytes = (compact)
+ ? auxHashMap.getCompactSizeBytes()
+ : auxHashMap.getUpdatableSizeBytes();
+ } else {
+ auxBytes = (compact) ? 0 : 4 << LG_AUX_ARR_INTS[impl.lgConfigK];
+ }
+ }
+ final int totBytes = HLL_BYTE_ARR_START + impl.getHllByteArrBytes() + auxBytes;
+ final byte[] byteArr = new byte[totBytes];
+ final MemorySegment wseg = MemorySegment.ofArray(byteArr);
+ insertHll(impl, wseg, compact);
+ return byteArr;
+ }
+
+ private static final void insertHll(final AbstractHllArray impl, final MemorySegment wseg,
+ final boolean compact) {
+ insertCommonHll(impl, wseg, compact);
+ final byte[] hllByteArr = ((HllArray)impl).hllByteArr;
+ MemorySegment.copy(hllByteArr, 0, wseg, JAVA_BYTE, HLL_BYTE_ARR_START, hllByteArr.length);
+ if (impl.getAuxHashMap() != null) {
+ insertAux(impl, wseg, compact);
+ } else {
+ wseg.set(JAVA_INT_UNALIGNED, AUX_COUNT_INT, 0);
+ }
+ }
+
+ private static final void insertCommonHll(final AbstractHllArray srcImpl,
+ final MemorySegment tgtWseg, final boolean compact) {
+ insertPreInts(tgtWseg, srcImpl.getPreInts());
+ insertSerVer(tgtWseg);
+ insertFamilyId(tgtWseg);
+ insertLgK(tgtWseg, srcImpl.getLgConfigK());
+ insertEmptyFlag(tgtWseg, srcImpl.isEmpty());
+ insertCompactFlag(tgtWseg, compact);
+ insertOooFlag(tgtWseg, srcImpl.isOutOfOrder());
+ insertCurMin(tgtWseg, srcImpl.getCurMin());
+ insertCurMode(tgtWseg, srcImpl.getCurMode());
+ insertTgtHllType(tgtWseg, srcImpl.getTgtHllType());
+ insertHipAccum(tgtWseg, srcImpl.getHipAccum());
+ insertKxQ0(tgtWseg, srcImpl.getKxQ0());
+ insertKxQ1(tgtWseg, srcImpl.getKxQ1());
+ insertNumAtCurMin(tgtWseg, srcImpl.getNumAtCurMin());
+ insertRebuildCurMinNumKxQFlag(tgtWseg, srcImpl.isRebuildCurMinNumKxQFlag());
+ }
+
+ private static final void insertAux(final AbstractHllArray srcImpl, final MemorySegment tgtWseg,
+ final boolean tgtCompact) {
+ final AuxHashMap auxHashMap = srcImpl.getAuxHashMap();
+ final int auxCount = auxHashMap.getAuxCount();
+ insertAuxCount(tgtWseg, auxCount);
+ insertLgArr(tgtWseg, auxHashMap.getLgAuxArrInts()); //only used for direct HLL
+ final long auxStart = srcImpl.auxStart;
+ if (tgtCompact) {
+ final PairIterator itr = auxHashMap.getIterator();
+ int cnt = 0;
+ while (itr.nextValid()) { //works whether src has compact MemorySegment or not
+ insertInt(tgtWseg, auxStart + (cnt++ << 2), itr.getPair());
+ }
+ assert cnt == auxCount;
+ } else { //updatable
+ final int auxInts = 1 << auxHashMap.getLgAuxArrInts();
+ final int[] auxArr = auxHashMap.getAuxIntArr();
+ MemorySegment.copy(auxArr, 0, tgtWseg, JAVA_INT_UNALIGNED, auxStart, auxInts);
+ }
+ }
+
+ //To byte array for coupons
+ static final byte[] toCouponByteArray(final AbstractCoupons impl, final boolean dstCompact) {
+ final int srcCouponCount = impl.getCouponCount();
+ final int srcLgCouponArrInts = impl.getLgCouponArrInts();
+ final int srcCouponArrInts = 1 << srcLgCouponArrInts;
+ final byte[] byteArrOut;
+ final boolean list = impl.getCurMode() == CurMode.LIST;
+ //prepare switch
+ final int sw = (impl.hasMemorySegment() ? 0 : 4) | (impl.isCompact() ? 0 : 2) | (dstCompact ? 0 : 1);
+ switch (sw) {
+ case 0: { //Src MemorySegment, Src Compact, Dst Compact
+ final MemorySegment srcSeg = impl.getMemorySegment();
+ final int bytesOut = impl.getSegDataStart() + (srcCouponCount << 2);
+ byteArrOut = new byte[bytesOut];
+ MemorySegment.copy(srcSeg, JAVA_BYTE, 0, byteArrOut, 0, bytesOut);
+ break;
+ }
+ case 1: { //Src MemorySegment, Src Compact, Dst Updatable
+ final int dataStart = impl.getSegDataStart();
+ final int bytesOut = dataStart + (srcCouponArrInts << 2);
+ byteArrOut = new byte[bytesOut];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArrOut);
+ copyCommonListAndSet(impl, segOut);
+ insertCompactFlag(segOut, dstCompact);
+
+ final int[] tgtCouponIntArr = new int[srcCouponArrInts];
+ final PairIterator itr = impl.iterator();
+ while (itr.nextValid()) {
+ final int pair = itr.getPair();
+ final int idx = find(tgtCouponIntArr, srcLgCouponArrInts, pair);
+ if (idx < 0) { //found EMPTY
+ tgtCouponIntArr[~idx] = pair; //insert
+ continue;
+ }
+ throw new SketchesStateException("Error: found duplicate.");
+ }
+ MemorySegment.copy(tgtCouponIntArr, 0, segOut, JAVA_INT_UNALIGNED, dataStart, srcCouponArrInts);
+
+ if (list) {
+ insertListCount(segOut, srcCouponCount);
+ } else {
+ insertHashSetCount(segOut, srcCouponCount);
+ }
+ break;
+ }
+
+ case 6: //Src Heap, Src Updatable, Dst Compact
+ case 2: { //Src MemorySegment, Src Updatable, Dst Compact
+ final int dataStart = impl.getSegDataStart();
+ final int bytesOut = dataStart + (srcCouponCount << 2);
+ byteArrOut = new byte[bytesOut];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArrOut);
+ copyCommonListAndSet(impl, segOut);
+ insertCompactFlag(segOut, dstCompact);
+
+ final PairIterator itr = impl.iterator();
+ int cnt = 0;
+ while (itr.nextValid()) {
+ insertInt(segOut, dataStart + (cnt++ << 2), itr.getPair());
+ }
+ if (list) {
+ insertListCount(segOut, srcCouponCount);
+ } else {
+ insertHashSetCount(segOut, srcCouponCount);
+ }
+ break;
+ }
+ case 3: { //Src MemorySegment, Src Updatable, Dst Updatable
+ final MemorySegment srcSeg = impl.getMemorySegment();
+ final int bytesOut = impl.getSegDataStart() + (srcCouponArrInts << 2);
+ byteArrOut = new byte[bytesOut];
+ MemorySegment.copy(srcSeg, JAVA_BYTE, 0, byteArrOut, 0, bytesOut);
+ break;
+ }
+ case 7: { //Src Heap, Src Updatable, Dst Updatable
+ final int dataStart = impl.getSegDataStart();
+ final int bytesOut = dataStart + (srcCouponArrInts << 2);
+ byteArrOut = new byte[bytesOut];
+ final MemorySegment segOut = MemorySegment.ofArray(byteArrOut);
+ copyCommonListAndSet(impl, segOut);
+
+ MemorySegment.copy(impl.getCouponIntArr(), 0, segOut, JAVA_INT_UNALIGNED, dataStart, srcCouponArrInts);
+ if (list) {
+ insertListCount(segOut, srcCouponCount);
+ } else {
+ insertHashSetCount(segOut, srcCouponCount);
+ }
+ break;
+ }
+ default: throw new SketchesStateException("Corruption, should not happen: " + sw);
+ }
+ return byteArrOut;
+ }
+
+ private static final void copyCommonListAndSet(final AbstractCoupons impl,
+ final MemorySegment wseg) {
+ insertPreInts(wseg, impl.getPreInts());
+ insertSerVer(wseg);
+ insertFamilyId(wseg);
+ insertLgK(wseg, impl.getLgConfigK());
+ insertLgArr(wseg, impl.getLgCouponArrInts());
+ insertEmptyFlag(wseg, impl.isEmpty());
+ insertOooFlag(wseg, impl.isOutOfOrder());
+ insertCurMode(wseg, impl.getCurMode());
+ insertTgtHllType(wseg, impl.getTgtHllType());
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/Union.java b/src/main/java/org/apache/datasketches/hll2/Union.java
new file mode 100644
index 000000000..395526954
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/Union.java
@@ -0,0 +1,826 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.apache.datasketches.common.Util.invPow2;
+import static org.apache.datasketches.hll2.HllUtil.AUX_TOKEN;
+import static org.apache.datasketches.hll2.HllUtil.EMPTY;
+import static org.apache.datasketches.hll2.HllUtil.loNibbleMask;
+import static org.apache.datasketches.hll2.PreambleUtil.HLL_BYTE_ARR_START;
+import static org.apache.datasketches.hll2.PreambleUtil.extractTgtHllType;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_4;
+import static org.apache.datasketches.hll2.TgtHllType.HLL_8;
+
+import java.lang.foreign.MemorySegment;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+
+/**
+ * This performs union operations for all HllSketches. This union operator can be configured to be
+ * on or off heap. The source sketch given to this union using the {@link #update(HllSketch)} can
+ * be configured with any precision value <i>lgConfigK</i> (from 4 to 21), any <i>TgtHllType</i>
+ * (HLL_4, HLL_6, HLL_8), and either on or off-heap; and it can be in either of the sparse modes
+ * (<i>LIST</i> or <i>SET</i>), or the dense mode (<i>HLL</i>).
+ *
+ * <p>Although the API for this union operator parallels many of the methods of the
+ * <i>HllSketch</i>, the behavior of the union operator has some fundamental differences.</p>
+ *
+ * <p>First, this union operator is configured with a <i>lgMaxK</i> instead of the normal
+ * <i>lgConfigK</i>. Generally, this union operator will inherit the lowest <i>lgConfigK</i>
+ * less than <i>lgMaxK</i> that it has seen. However, the <i>lgConfigK</i> of incoming sketches that
+ * are still in sparse are ignored. The <i>lgMaxK</i> provides the user the ability to specify the
+ * largest maximum size for the union operation.
+ *
+ * <p>Second, the user cannot specify the {@link TgtHllType} as an input parameter to the union.
+ * Instead, it is specified for the sketch returned with {@link #getResult(TgtHllType)}.
+ *
+ * <p>The following graph illustrates the HLL Merge speed.</p>
+ *
+ * <p><img src="doc-files/HLL_UnionTime4_6_8_Java_CPP.png" width="500" alt="HLL LgK12 Union Speed"></p>
+ * This graph illustrates the relative merging speed of the HLL 4,6,8 Java HLL sketches compared to
+ * the DataSketches C++ implementations of the same sketches. With this particular test (merging 32 relative large
+ * sketches together), the Java HLL 8 is the fastest and the Java HLL 4 the slowest, with a mixed cluster in the middle.
+ * Union / Merging speed is somewhat difficult to measure as the performance is very dependent on the mix of sketch
+ * sizes (and types) you are merging. So your mileage will vary!
+ *
+ * <p>For a complete example of using the Union operator
+ * see <a href="https://datasketches.apache.org/docs/HLL/HllJavaExample.html">Union Example</a>.</p>
+ *
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+public class Union extends BaseHllSketch {
+ final int lgMaxK;
+ private final HllSketch gadget;
+
+ /**
+ * Construct this Union operator with the default maximum log-base-2 of <i>K</i>.
+ */
+ public Union() {
+ lgMaxK = HllSketch.DEFAULT_LG_K;
+ gadget = new HllSketch(lgMaxK, HLL_8);
+ }
+
+ /**
+ * Construct this Union operator with a given maximum log-base-2 of <i>K</i>.
+ * @param lgMaxK the desired maximum log-base-2 of <i>K</i>. This value must be
+ * between 4 and 21 inclusively.
+ */
+ public Union(final int lgMaxK) {
+ this.lgMaxK = HllUtil.checkLgK(lgMaxK);
+ gadget = new HllSketch(lgMaxK, HLL_8);
+ }
+
+ /**
+ * Construct this Union operator with a given maximum log-base-2 of <i>K</i> and the given
+ * MemorySegment as the destination for this Union. This MemorySegment is usually configured
+ * for off-heap MemorySegment. What remains on the java heap is a thin wrapper object that reads and
+ * writes to the given MemorySegment.
+ *
+ * <p>The given <i>dstSeg</i> is checked for the required capacity as determined by
+ * {@link HllSketch#getMaxUpdatableSerializationBytes(int, TgtHllType)}.
+ * @param lgMaxK the desired maximum log-base-2 of <i>K</i>. This value must be
+ * between 4 and 21 inclusively.
+ * @param dstWseg the destination writable MemorySegment for the sketch.
+ */
+ public Union(final int lgMaxK, final MemorySegment dstWseg) {
+ this.lgMaxK = HllUtil.checkLgK(lgMaxK);
+ gadget = new HllSketch(lgMaxK, HLL_8, dstWseg);
+ }
+
+ //used only by writableWrap
+ private Union(final HllSketch sketch) {
+ lgMaxK = sketch.getLgConfigK();
+ gadget = sketch;
+ }
+
+ /**
+ * Construct a union operator populated with the given byte array image of an HllSketch.
+ * @param byteArray the given byte array
+ * @return a union operator populated with the given byte array image of an HllSketch.
+ */
+ public static final Union heapify(final byte[] byteArray) {
+ return heapify(MemorySegment.ofArray(byteArray));
+ }
+
+ /**
+ * Construct a union operator populated with the given MemorySegment image of an HllSketch.
+ * @param seg the given MemorySegment
+ * @return a union operator populated with the given MemorySegment image of an HllSketch.
+ */
+ public static final Union heapify(final MemorySegment seg) {
+ final int lgK = HllUtil.checkLgK(seg.get(JAVA_BYTE, PreambleUtil.LG_K_BYTE));
+ final HllSketch sk = HllSketch.heapify(seg, false); //allows non-finalized image
+ final Union union = new Union(lgK);
+ union.update(sk);
+ return union;
+ }
+
+ /**
+ * Wraps the given MemorySegment, which must be a image of a valid updatable HLL_8 sketch,
+ * and may have data. What remains on the java heap is a
+ * thin wrapper object that reads and writes to the given MemorySegment, which, depending on
+ * how the user configures the MemorySegment, may actually reside on the Java heap or off-heap.
+ *
+ * <p>The given <i>dstSeg</i> is checked for the required capacity as determined by
+ * {@link HllSketch#getMaxUpdatableSerializationBytes(int, TgtHllType)}, and for the correct type.
+ * @param srcWseg an writable image of a valid sketch with data.
+ * @return a Union operator where the sketch data is in the given dstSeg.
+ */
+ public static final Union writableWrap(final MemorySegment srcWseg) {
+ final TgtHllType tgtHllType = extractTgtHllType(srcWseg);
+ if (tgtHllType != TgtHllType.HLL_8) {
+ throw new SketchesArgumentException(
+ "Union can only wrap writable HLL_8 sketches that were the Gadget of a Union.");
+ }
+ //allows writableWrap of non-finalized image
+ return new Union(HllSketch.writableWrap(srcWseg, false));
+ }
+
+ @Override
+ public double getCompositeEstimate() {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.hllSketchImpl.getCompositeEstimate();
+ }
+
+ @Override
+ CurMode getCurMode() {
+ return gadget.getCurMode();
+ }
+
+ @Override
+ public int getCompactSerializationBytes() {
+ return gadget.getCompactSerializationBytes();
+ }
+
+ @Override
+ public double getEstimate() {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.getEstimate();
+ }
+
+ /**
+ * Gets the effective <i>lgConfigK</i> for the union operator, which may be less than
+ * <i>lgMaxK</i>.
+ * @return the <i>lgConfigK</i>.
+ */
+ @Override
+ public int getLgConfigK() {
+ return gadget.getLgConfigK();
+ }
+
+ @Override
+ public double getLowerBound(final int numStdDev) {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.getLowerBound(numStdDev);
+ }
+
+ /**
+ * Returns the maximum size in bytes that this union operator can grow to given a lgK.
+ *
+ * @param lgK The maximum Log2 of K for this union operator. This value must be
+ * between 4 and 21 inclusively.
+ * @return the maximum size in bytes that this union operator can grow to.
+ */
+ public static int getMaxSerializationBytes(final int lgK) {
+ return HllSketch.getMaxUpdatableSerializationBytes(lgK, TgtHllType.HLL_8);
+ }
+
+ /**
+ * Return the result of this union operator as an HLL_4 sketch.
+ * @return the result of this union operator as an HLL_4 sketch.
+ */
+ public HllSketch getResult() {
+ return getResult(HllSketch.DEFAULT_HLL_TYPE);
+ }
+
+ /**
+ * Return the result of this union operator with the specified {@link TgtHllType}
+ * @param tgtHllType the TgtHllType enum
+ * @return the result of this union operator with the specified TgtHllType
+ */
+ public HllSketch getResult(final TgtHllType tgtHllType) {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.copyAs(tgtHllType);
+ }
+
+ @Override
+ public TgtHllType getTgtHllType() {
+ return TgtHllType.HLL_8;
+ }
+
+ @Override
+ public int getUpdatableSerializationBytes() {
+ return gadget.getUpdatableSerializationBytes();
+ }
+
+ @Override
+ public double getUpperBound(final int numStdDev) {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.getUpperBound(numStdDev);
+ }
+
+ @Override
+ public boolean isCompact() {
+ return gadget.isCompact();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return gadget.isEmpty();
+ }
+
+ @Override
+ public boolean hasMemorySegment() {
+ return gadget.hasMemorySegment();
+ }
+
+ @Override
+ public boolean isOffHeap() {
+ return gadget.isOffHeap();
+ }
+
+ @Override
+ boolean isOutOfOrder() {
+ return gadget.isOutOfOrder();
+ }
+
+ @Override
+ public boolean isSameResource(final MemorySegment seg) {
+ return gadget.isSameResource(seg);
+ }
+
+ boolean isRebuildCurMinNumKxQFlag() {
+ return gadget.hllSketchImpl.isRebuildCurMinNumKxQFlag();
+ }
+
+ void putRebuildCurMinNumKxQFlag(final boolean rebuild) {
+ gadget.hllSketchImpl.putRebuildCurMinNumKxQFlag(rebuild);
+ }
+
+ /**
+ * Resets to empty and retains the current lgK, but does not change the configured value of
+ * lgMaxK.
+ */
+ @Override
+ public void reset() {
+ gadget.reset();
+ }
+
+ /**
+ * Gets the serialization of this union operator as a byte array in compact form, which is
+ * designed to be heapified only. It is not directly updatable.
+ * For the Union operator, this is the serialization of the internal state of
+ * the union operator as a sketch.
+ * @return the serialization of this union operator as a byte array.
+ */
+ @Override
+ public byte[] toCompactByteArray() {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.toCompactByteArray();
+ }
+
+ @Override
+ public byte[] toUpdatableByteArray() {
+ checkRebuildCurMinNumKxQ(gadget);
+ return gadget.toUpdatableByteArray();
+ }
+
+ @Override
+ public String toString(final boolean summary, final boolean hllDetail,
+ final boolean auxDetail, final boolean all) {
+ final HllSketch clone = gadget.copy();
+ checkRebuildCurMinNumKxQ(clone);
+ return clone.toString(summary, hllDetail, auxDetail, all);
+ }
+
+ /**
+ * Update this union operator with the given sketch.
+ * @param sketch the given sketch.
+ */
+ public void update(final HllSketch sketch) {
+ gadget.hllSketchImpl = unionImpl(sketch, gadget, lgMaxK);
+ }
+
+ @Override
+ void couponUpdate(final int coupon) {
+ if (coupon == EMPTY) { return; }
+ gadget.hllSketchImpl = gadget.hllSketchImpl.couponUpdate(coupon);
+ }
+
+ // Union operator logic
+
+ /**
+ * Union the given source and destination sketches. This static method examines the state of
+ * the current internal gadget and the incoming sketch and determines the optimum way to
+ * perform the union. This may involve swapping the merge order, downsampling, transforming,
+ * and / or copying one of the arguments and may completely replace the internals of the union.
+ *
+ * <p>If the union gadget is empty, the source sketch is effectively copied to the union gadget
+ * after any required transformations.
+ *
+ * <p>The direction of the merge is reversed if the union gadget is in LIST or SET mode, and the
+ * source sketch is in HLL mode. This is done to maintain maximum accuracy of the union process.
+ *
+ * <p>The source sketch is downsampled if the source LgK is larger than maxLgK and in HLL mode.
+ *
+ * <p>The union gadget is downsampled if both source and union gadget are in HLL mode
+ * and the source LgK <b>less than</b> the union gadget LgK.
+ *
+ * @param source the given incoming sketch, which cannot be modified.
+ * @param gadget the given gadget sketch, which has a target of HLL_8 and holds the result.
+ * @param lgMaxK the maximum value of log2 K for this union.
+ * @return the union of the two sketches in the form of the internal HllSketchImpl, which is
+ * always in HLL_8 form.
+ */
+ private static HllSketchImpl unionImpl(final HllSketch source, final HllSketch gadget,
+ final int lgMaxK) {
+ assert gadget.getTgtHllType() == HLL_8;
+ if ((source == null) || source.isEmpty()) {
+ return gadget.hllSketchImpl;
+ }
+
+ final CurMode srcMode = source.getCurMode();
+ if (srcMode == CurMode.LIST ) {
+ source.mergeTo(gadget);
+ return gadget.hllSketchImpl;
+ }
+
+ final int srcLgK = source.getLgConfigK();
+ final int gadgetLgK = gadget.getLgConfigK();
+ final boolean srcHasSeg = source.hasMemorySegment();
+ final boolean gdtHasSeg = gadget.hasMemorySegment();
+ final boolean gdtEmpty = gadget.isEmpty();
+
+ if (srcMode == CurMode.SET ) {
+ if (gdtEmpty && (srcLgK == gadgetLgK) && (!srcHasSeg) && (!gdtHasSeg)) {
+ gadget.hllSketchImpl = source.copyAs(HLL_8).hllSketchImpl;
+ return gadget.hllSketchImpl;
+ }
+ source.mergeTo(gadget);
+ return gadget.hllSketchImpl;
+ }
+
+ //Hereafter, the source is in HLL mode.
+ final int bit0 = gdtHasSeg ? 1 : 0;
+ final int bits1_2 = (gdtEmpty ? 3 : gadget.getCurMode().ordinal()) << 1;
+ final int bit3 = (srcLgK < gadgetLgK) ? 8 : 0;
+ final int bit4 = (srcLgK > lgMaxK) ? 16 : 0;
+ final int sw = bit4 | bit3 | bits1_2 | bit0;
+ HllSketchImpl hllSketchImpl = null; //never returned as null
+
+ switch (sw) {
+ case 0: //src <= max, src >= gdt, gdtLIST, gdtHeap
+ case 8: //src <= max, src < gdt, gdtLIST, gdtHeap
+ case 2: //src <= max, src >= gdt, gdtSET, gdtHeap
+ case 10://src <= max, src < gdt, gdtSET, gdtHeap
+ { //Action: copy src, reverse merge w/autofold, ooof=src
+ final HllSketch srcHll8Heap = source.copyAs(HLL_8);
+ gadget.mergeTo(srcHll8Heap); //merge gdt(Hll8,heap,list/set) -> src(Hll8,heap,hll)
+ hllSketchImpl = srcHll8Heap.hllSketchImpl;
+ break;
+ }
+ case 16://src > max, src >= gdt, gdtList, gdtHeap
+ case 18://src > max, src >= gdt, gdtSet, gdtHeap
+ { //Action: downsample src to MaxLgK, reverse merge w/autofold, ooof=src
+ final HllSketch srcHll8Heap = downsample(source, lgMaxK);
+ gadget.mergeTo(srcHll8Heap); //merge gdt(Hll8,heap,list/set) -> src(Hll8,heap,hll)
+ hllSketchImpl = srcHll8Heap.hllSketchImpl;
+ break;
+ }
+
+ case 1: //src <= max, src >= gdt, gdtLIST, gdtMemorySegment
+ case 9: //src <= max, src < gdt, gdtLIST, gdtMemorySegment
+ case 3: //src <= max, src >= gdt, gdtSET, gdtMemorySegment
+ case 11://src <= max, src < gdt, gdtSET, gdtMemorySegment
+ { //Action: copy src, reverse merge w/autofold, use gdt MemorySegment, ooof=src
+ final HllSketch srcHll8Heap = source.copyAs(HLL_8);
+ gadget.mergeTo(srcHll8Heap); //merge gdt(Hll8,seg,list/set) -> src(Hll8,heap,hll)
+ hllSketchImpl = useGadgetMemorySegment(gadget, srcHll8Heap, false).hllSketchImpl;
+ break;
+ }
+ case 17://src > max, src >= gdt, gdtList, gdtMemorySegment
+ case 19://src > max, src >= gdt, gdtSet, gdtMemorySegment
+ { //Action: downsample src to MaxLgK, reverse merge w/autofold, use gdt MemorySegment, ooof=src
+ final HllSketch srcHll8Heap = downsample(source, lgMaxK);
+ gadget.mergeTo(srcHll8Heap); //merge gdt(Hll8,seg,list/set) -> src(Hll8,heap,hll), autofold
+ hllSketchImpl = useGadgetMemorySegment(gadget, srcHll8Heap, false).hllSketchImpl;
+ break;
+ }
+
+ case 4: //src <= max, src >= gdt, gdtHLL, gdtHeap
+ case 20://src > max, src >= gdt, gdtHLL, gdtHeap
+ case 5: //src <= max, src >= gdt, gdtHLL, gdtMemorySegment
+ case 21://src > max, src >= gdt, gdtHLL, gdtMemorySegment
+ { //Action: forward HLL merge w/autofold, ooof=True
+ //merge src(Hll4,6,8,heap/seg,Mode=HLL) -> gdt(Hll8,heap,Mode=HLL)
+ mergeHlltoHLLmode(source, gadget, srcLgK, gadgetLgK, srcHasSeg, gdtHasSeg);
+ hllSketchImpl = gadget.putOutOfOrderFlag(true).hllSketchImpl;
+ break;
+ }
+ case 12://src <= max, src < gdt, gdtHLL, gdtHeap
+ { //Action: downsample gdt to srcLgK, forward HLL merge w/autofold, ooof=True
+ final HllSketch gdtHll8Heap = downsample(gadget, srcLgK);
+ //merge src(Hll4,6,8;heap/seg,Mode=HLL) -> gdt(Hll8,heap,hll)
+ mergeHlltoHLLmode(source, gdtHll8Heap, srcLgK, gadgetLgK, srcHasSeg, false);
+ hllSketchImpl = gdtHll8Heap.putOutOfOrderFlag(true).hllSketchImpl;
+ break;
+ }
+ case 13://src <= max, src < gdt, gdtHLL, gdtMemorySegment
+ { //Action: downsample gdt to srcLgK, forward HLL merge w/autofold, use gdt MemorySegment, ooof=True
+ final HllSketch gdtHll8Heap = downsample(gadget, srcLgK);
+ //merge src(Hll4,6,8;heap/seg;Mode=HLL) -> gdt(Hll8,heap,Mode=HLL)
+ mergeHlltoHLLmode(source, gdtHll8Heap, srcLgK, gadgetLgK, srcHasSeg, false);
+ hllSketchImpl = useGadgetMemorySegment(gadget, gdtHll8Heap, true).hllSketchImpl;
+ break;
+ }
+
+ case 6: //src <= max, src >= gdt, gdtEmpty, gdtHeap
+ case 14://src <= max, src < gdt, gdtEmpty, gdtHeap
+ { //Action: copy src, replace gdt, ooof=src
+ final HllSketch srcHll8Heap = source.copyAs(HLL_8);
+ hllSketchImpl = srcHll8Heap.hllSketchImpl;
+ break;
+ }
+ case 22://src > max, src >= gdt, gdtEmpty, gdtHeap
+ { //Action: downsample src to lgMaxK, replace gdt, ooof=src
+ final HllSketch srcHll8Heap = downsample(source, lgMaxK);
+ hllSketchImpl = srcHll8Heap.hllSketchImpl;
+ break;
+ }
+
+ case 7: //src <= max, src >= gdt, gdtEmpty, gdtMemorySegment
+ case 15://src <= max, src < gdt, gdtEmpty, gdtMemorySegment
+ { //Action: copy src, use gdt MemorySegment, ooof=src
+ final HllSketch srcHll8Heap = source.copyAs(HLL_8);
+ hllSketchImpl = useGadgetMemorySegment(gadget, srcHll8Heap, false).hllSketchImpl;
+ break;
+ }
+ case 23://src > max, src >= gdt, gdtEmpty, gdtMemorySegment, replace seg, downsample src, ooof=src
+ { //Action: downsample src to lgMaxK, use gdt MemorySegment, ooof=src
+ final HllSketch srcHll8Heap = downsample(source, lgMaxK);
+ hllSketchImpl = useGadgetMemorySegment(gadget, srcHll8Heap, false).hllSketchImpl;
+ break;
+ }
+ default: return gadget.hllSketchImpl; //not possible
+ }
+ return hllSketchImpl;
+ }
+
+ private static final HllSketch useGadgetMemorySegment(
+ final HllSketch gadget, final HllSketch hll8Heap, final boolean setOooFlag) {
+ final MemorySegment wseg = gadget.getMemorySegment(); //use the gdt wseg
+ final byte[] byteArr = hll8Heap.toUpdatableByteArray(); //serialize srcCopy
+ MemorySegment.copy(byteArr, 0, wseg, JAVA_BYTE, 0, byteArr.length); //replace old data with new
+ return (setOooFlag)
+ ? HllSketch.writableWrap(wseg, false).putOutOfOrderFlag(true) //wrap, set oooflag, return
+ : HllSketch.writableWrap(wseg, false); //wrap & return
+ }
+
+ private static final void mergeHlltoHLLmode(final HllSketch src, final HllSketch tgt,
+ final int srcLgK, final int tgtLgK, final boolean srcHasSeg, final boolean tgtHasSeg) {
+ final int sw = (tgtHasSeg ? 1 : 0) | (srcHasSeg ? 2 : 0)
+ | ((srcLgK > tgtLgK) ? 4 : 0) | ((src.getTgtHllType() != HLL_8) ? 8 : 0);
+ final int srcK = 1 << srcLgK;
+
+ switch (sw) {
+ case 0: { //HLL_8, srcLgK=tgtLgK, src=heap, tgt=heap
+ final byte[] srcArr = ((Hll8Array) src.hllSketchImpl).hllByteArr;
+ final byte[] tgtArr = ((Hll8Array) tgt.hllSketchImpl).hllByteArr;
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcArr[i];
+ final byte tgtV = tgtArr[i];
+ tgtArr[i] = (byte) Math.max(srcV, tgtV);
+ }
+ break;
+ }
+ case 1: { //HLL_8, srcLgK=tgtLgK, src=heap, tgt=seg
+ final byte[] srcArr = ((Hll8Array) src.hllSketchImpl).hllByteArr;
+ final MemorySegment tgtSeg = tgt.getMemorySegment();
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcArr[i];
+ final byte tgtV = tgtSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ tgtSeg.set(JAVA_BYTE, HLL_BYTE_ARR_START + i, (byte) Math.max(srcV, tgtV));
+ }
+ break;
+ }
+ case 2: { //HLL_8, srcLgK=tgtLgK, src=seg, tgt=heap
+ final MemorySegment srcSeg = src.getMemorySegment();
+ final byte[] tgtArr = ((Hll8Array) tgt.hllSketchImpl).hllByteArr;
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ final byte tgtV = tgtArr[i];
+ tgtArr[i] = (byte) Math.max(srcV, tgtV);
+ }
+ break;
+ }
+ case 3: { //HLL_8, srcLgK=tgtLgK, src=seg, tgt=seg
+ final MemorySegment srcSeg = src.getMemorySegment();
+ final MemorySegment tgtSeg = tgt.getMemorySegment();
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ final byte tgtV = tgtSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ tgtSeg.set(JAVA_BYTE, HLL_BYTE_ARR_START + i, (byte) Math.max(srcV, tgtV));
+ }
+ break;
+ }
+ case 4: { //HLL_8, srcLgK>tgtLgK, src=heap, tgt=heap
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final byte[] srcArr = ((Hll8Array) src.hllSketchImpl).hllByteArr;
+ final byte[] tgtArr = ((Hll8Array) tgt.hllSketchImpl).hllByteArr;
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcArr[i];
+ final int j = i & tgtKmask;
+ final byte tgtV = tgtArr[j];
+ tgtArr[j] = (byte) Math.max(srcV, tgtV);
+ }
+ break;
+ }
+ case 5: { //HLL_8, srcLgK>tgtLgK, src=heap, tgt=seg
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final byte[] srcArr = ((Hll8Array) src.hllSketchImpl).hllByteArr;
+ final MemorySegment tgtSeg = tgt.getMemorySegment();
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcArr[i];
+ final int j = i & tgtKmask;
+ final byte tgtV = tgtSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + j);
+ tgtSeg.set(JAVA_BYTE, HLL_BYTE_ARR_START + j, (byte) Math.max(srcV, tgtV));
+ }
+ break;
+ }
+ case 6: { //HLL_8, srcLgK>tgtLgK, src=seg, tgt=heap
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final MemorySegment srcSeg = src.getMemorySegment();
+ final byte[] tgtArr = ((Hll8Array) tgt.hllSketchImpl).hllByteArr;
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ final int j = i & tgtKmask;
+ final byte tgtV = tgtArr[j];
+ tgtArr[j] = (byte) Math.max(srcV, tgtV);
+ }
+ break;
+ }
+ case 7: { //HLL_8, srcLgK>tgtLgK, src=seg, tgt=seg
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final MemorySegment srcSeg = src.getMemorySegment();
+ final MemorySegment tgtSeg = tgt.getMemorySegment();
+ for (int i = 0; i < srcK; i++) {
+ final byte srcV = srcSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i);
+ final int j = i & tgtKmask;
+ final byte tgtV = tgtSeg.get(JAVA_BYTE, HLL_BYTE_ARR_START + j);
+ tgtSeg.set(JAVA_BYTE, HLL_BYTE_ARR_START + j, (byte) Math.max(srcV, tgtV));
+ }
+ break;
+ }
+ case 8: case 9:
+ {
+ //!HLL_8, srcLgK=tgtLgK, src=heap, tgt=heap/seg
+ final AbstractHllArray tgtAbsHllArr = (AbstractHllArray)(tgt.hllSketchImpl);
+ if (src.getTgtHllType() == HLL_4) {
+ final Hll4Array src4 = (Hll4Array) src.hllSketchImpl;
+ final AuxHashMap auxHashMap = src4.getAuxHashMap();
+ final int curMin = src4.getCurMin();
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b = src4.hllByteArr[i++];
+ int value = Byte.toUnsignedInt(b) & loNibbleMask;
+ tgtAbsHllArr.updateSlotNoKxQ(j, value == AUX_TOKEN ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ value = Byte.toUnsignedInt(b) >>> 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j, value == AUX_TOKEN ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ }
+ } else {
+ final Hll6Array src6 = (Hll6Array) src.hllSketchImpl;
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b1 = src6.hllByteArr[i++];
+ final byte b2 = src6.hllByteArr[i++];
+ final byte b3 = src6.hllByteArr[i++];
+ int value = Byte.toUnsignedInt(b1) & 0x3f;
+ tgtAbsHllArr.updateSlotNoKxQ(j++, value);
+ value = Byte.toUnsignedInt(b1) >>> 6;
+ value |= (Byte.toUnsignedInt(b2) & 0x0f) << 2;
+ tgtAbsHllArr.updateSlotNoKxQ(j++, value);
+ value = Byte.toUnsignedInt(b2) >>> 4;
+ value |= (Byte.toUnsignedInt(b3) & 3) << 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j++, value);
+ value = Byte.toUnsignedInt(b3) >>> 2;
+ tgtAbsHllArr.updateSlotNoKxQ(j++, value);
+ }
+ }
+ break;
+ }
+ case 10: case 11:
+ {
+ //!HLL_8, srcLgK=tgtLgK, src=seg, tgt=heap/seg
+ final AbstractHllArray tgtAbsHllArr = (AbstractHllArray)(tgt.hllSketchImpl);
+ if (src.getTgtHllType() == HLL_4) {
+ final DirectHll4Array src4 = (DirectHll4Array) src.hllSketchImpl;
+ final AuxHashMap auxHashMap = src4.getAuxHashMap();
+ final int curMin = src4.getCurMin();
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b = src4.seg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i++);
+ int value = Byte.toUnsignedInt(b) & loNibbleMask;
+ tgtAbsHllArr.updateSlotNoKxQ(j, value == AUX_TOKEN ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ value = Byte.toUnsignedInt(b) >>> 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j, value == AUX_TOKEN ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ }
+ } else {
+ final DirectHll6Array src6 = (DirectHll6Array) src.hllSketchImpl;
+ int i = 0;
+ int offset = HLL_BYTE_ARR_START;
+ while (i < srcK) {
+ final byte b1 = src6.seg.get(JAVA_BYTE, offset++);
+ final byte b2 = src6.seg.get(JAVA_BYTE, offset++);
+ final byte b3 = src6.seg.get(JAVA_BYTE, offset++);
+ int value = Byte.toUnsignedInt(b1) & 0x3f;
+ tgtAbsHllArr.updateSlotNoKxQ(i++, value);
+ value = Byte.toUnsignedInt(b1) >>> 6;
+ value |= (Byte.toUnsignedInt(b2) & 0x0f) << 2;
+ tgtAbsHllArr.updateSlotNoKxQ(i++, value);
+ value = Byte.toUnsignedInt(b2) >>> 4;
+ value |= (Byte.toUnsignedInt(b3) & 3) << 4;
+ tgtAbsHllArr.updateSlotNoKxQ(i++, value);
+ value = Byte.toUnsignedInt(b3) >>> 2;
+ tgtAbsHllArr.updateSlotNoKxQ(i++, value);
+ }
+ }
+ break;
+ }
+ case 12: case 13:
+ {
+ //!HLL_8, srcLgK>tgtLgK, src=heap, tgt=heap/seg
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final AbstractHllArray tgtAbsHllArr = (AbstractHllArray)(tgt.hllSketchImpl);
+ if (src.getTgtHllType() == HLL_4) {
+ final Hll4Array src4 = (Hll4Array) src.hllSketchImpl;
+ final AuxHashMap auxHashMap = src4.getAuxHashMap();
+ final int curMin = src4.getCurMin();
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b = src4.hllByteArr[i++];
+ int value = Byte.toUnsignedInt(b) & loNibbleMask;
+ tgtAbsHllArr.updateSlotNoKxQ(j & tgtKmask, value == AUX_TOKEN
+ ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ value = Byte.toUnsignedInt(b) >>> 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j & tgtKmask, value == AUX_TOKEN
+ ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ }
+ } else {
+ final Hll6Array src6 = (Hll6Array) src.hllSketchImpl;
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b1 = src6.hllByteArr[i++];
+ final byte b2 = src6.hllByteArr[i++];
+ final byte b3 = src6.hllByteArr[i++];
+ int value = Byte.toUnsignedInt(b1) & 0x3f;
+ tgtAbsHllArr.updateSlotNoKxQ(j++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b1) >>> 6;
+ value |= (Byte.toUnsignedInt(b2) & 0x0f) << 2;
+ tgtAbsHllArr.updateSlotNoKxQ(j++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b2) >>> 4;
+ value |= (Byte.toUnsignedInt(b3) & 3) << 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b3) >>> 2;
+ tgtAbsHllArr.updateSlotNoKxQ(j++ & tgtKmask, value);
+ }
+ }
+ break;
+ }
+ case 14: case 15:
+ {
+ //!HLL_8, srcLgK>tgtLgK, src=seg, tgt=heap/seg
+ final int tgtKmask = (1 << tgtLgK) - 1;
+ final AbstractHllArray tgtAbsHllArr = (AbstractHllArray)(tgt.hllSketchImpl);
+ if (src.getTgtHllType() == HLL_4) {
+ final DirectHll4Array src4 = (DirectHll4Array) src.hllSketchImpl;
+ final AuxHashMap auxHashMap = src4.getAuxHashMap();
+ final int curMin = src4.getCurMin();
+ int i = 0;
+ int j = 0;
+ while (j < srcK) {
+ final byte b = src4.seg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i++);
+ int value = Byte.toUnsignedInt(b) & loNibbleMask;
+ tgtAbsHllArr.updateSlotNoKxQ(j & tgtKmask, value == AUX_TOKEN
+ ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ value = Byte.toUnsignedInt(b) >>> 4;
+ tgtAbsHllArr.updateSlotNoKxQ(j & tgtKmask, value == AUX_TOKEN
+ ? auxHashMap.mustFindValueFor(j) : value + curMin);
+ j++;
+ }
+ } else {
+ final DirectHll6Array src6 = (DirectHll6Array) src.hllSketchImpl;
+ int i = 0;
+ int offset = HLL_BYTE_ARR_START;
+ while (i < srcK) {
+ final byte b1 = src6.seg.get(JAVA_BYTE, offset++);
+ final byte b2 = src6.seg.get(JAVA_BYTE, offset++);
+ final byte b3 = src6.seg.get(JAVA_BYTE, offset++);
+ int value = Byte.toUnsignedInt(b1) & 0x3f;
+ tgtAbsHllArr.updateSlotNoKxQ(i++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b1) >>> 6;
+ value |= (Byte.toUnsignedInt(b2) & 0x0f) << 2;
+ tgtAbsHllArr.updateSlotNoKxQ(i++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b2) >>> 4;
+ value |= (Byte.toUnsignedInt(b3) & 3) << 4;
+ tgtAbsHllArr.updateSlotNoKxQ(i++ & tgtKmask, value);
+ value = Byte.toUnsignedInt(b3) >>> 2;
+ tgtAbsHllArr.updateSlotNoKxQ(i++ & tgtKmask, value);
+ }
+ }
+ break;
+ }
+ default: break; //not possible
+ }
+ tgt.hllSketchImpl.putRebuildCurMinNumKxQFlag(true);
+ }
+
+ //Used by union operator. Always copies or downsamples to Heap HLL_8.
+ //Caller must ultimately manage oooFlag, as caller has more context.
+ /**
+ * Copies or downsamples the given candidate HLLmode sketch to tgtLgK, HLL_8, on the heap.
+ *
+ * @param candidate the HllSketch to downsample, must be in HLL mode.
+ * @param tgtLgK the LgK to downsample to.
+ * @return the downsampled HllSketch.
+ */
+ private static final HllSketch downsample(final HllSketch candidate, final int tgtLgK) {
+ final AbstractHllArray candArr = (AbstractHllArray) candidate.hllSketchImpl;
+ final HllArray tgtHllArr = HllArray.newHeapHll(tgtLgK, TgtHllType.HLL_8);
+ final PairIterator candItr = candArr.iterator();
+ while (candItr.nextValid()) {
+ tgtHllArr.couponUpdate(candItr.getPair()); //rebuilds KxQ, etc.
+ }
+ //both of these are required for isomorphism
+ tgtHllArr.putHipAccum(candArr.getHipAccum());
+ tgtHllArr.putOutOfOrder(candidate.isOutOfOrder());
+ tgtHllArr.putRebuildCurMinNumKxQFlag(false);
+ return new HllSketch(tgtHllArr);
+ }
+
+ //Used to rebuild curMin, numAtCurMin and KxQ registers, due to high performance merge operation
+ static final void checkRebuildCurMinNumKxQ(final HllSketch sketch) {
+ final HllSketchImpl hllSketchImpl = sketch.hllSketchImpl;
+ final CurMode curMode = sketch.getCurMode();
+ final TgtHllType tgtHllType = sketch.getTgtHllType();
+ final boolean rebuild = hllSketchImpl.isRebuildCurMinNumKxQFlag();
+ if ( !rebuild || (curMode != CurMode.HLL) || (tgtHllType != HLL_8) ) { return; }
+ final AbstractHllArray absHllArr = (AbstractHllArray)(hllSketchImpl);
+ int curMin = 64;
+ int numAtCurMin = 0;
+ double kxq0 = 1 << absHllArr.getLgConfigK();
+ double kxq1 = 0;
+ final PairIterator itr = absHllArr.iterator();
+ while (itr.nextAll()) {
+ final int v = itr.getValue();
+ if (v > 0) {
+ if (v < 32) { kxq0 += invPow2(v) - 1.0; }
+ else { kxq1 += invPow2(v) - 1.0; }
+ }
+ if (v > curMin) { continue; }
+ if (v < curMin) {
+ curMin = v;
+ numAtCurMin = 1;
+ } else {
+ numAtCurMin++;
+ }
+ }
+ absHllArr.putKxQ0(kxq0);
+ absHllArr.putKxQ1(kxq1);
+ absHllArr.putCurMin(curMin);
+ absHllArr.putNumAtCurMin(numAtCurMin);
+ absHllArr.putRebuildCurMinNumKxQFlag(false);
+ //HipAccum is not affected
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_HIP_K12T20U20.png b/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_HIP_K12T20U20.png
new file mode 100644
index 000000000..7d3c8348f
Binary files /dev/null and b/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_HIP_K12T20U20.png differ
diff --git a/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_UnionTime4_6_8_Java_CPP.png b/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_UnionTime4_6_8_Java_CPP.png
new file mode 100644
index 000000000..23225887b
Binary files /dev/null and b/src/main/java/org/apache/datasketches/hll2/doc-files/HLL_UnionTime4_6_8_Java_CPP.png differ
diff --git a/src/main/java/org/apache/datasketches/hll2/package-info.java b/src/main/java/org/apache/datasketches/hll2/package-info.java
new file mode 100644
index 000000000..ec54ec96f
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll2/package-info.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * <h2>The DataSketches™ HLL sketch family package</h2>
+ * {@link org.apache.datasketches.hll.HllSketch HllSketch} and {@link org.apache.datasketches.hll.Union Union}
+ * are the public facing classes of this high performance implementation of Phillipe Flajolet's
+ * HyperLogLog algorithm[1] but with significantly improved error behavior and important features that can be
+ * essential for large production systems that must handle massive data.
+ *
+ * <h2>Key Features of the DataSketches™ HLL Sketch and its companion Union</h2>
+ *
+ * <h3>Advanced Estimation Algorithms for Optimum Accuracy</h3>
+ *
+ * <h4>Zero error at low cardinalities</h4>
+ * The HLL sketch leverages highly compact arrays and hash tables to keep exact counts until the transition to
+ * dense mode is required for space reasons. The result is perfect accuracy for very low cardinalities.
+ *
+ * <p>Accuracy for very small streams can be important because Big Data is often fragmented into millions of smaller
+ * streams (or segments) that inevitably are power-law distributed in size. If you are sketching all these fragments,
+ * as a general rule, more than 80% of your sketches will be very small, 20% will be much larger, and only a few very
+ * large in cardinality.
+ *
+ * <h4>HIP / Martingale Estimator</h4>
+ * When obtaining a cardinality estimate, the sketch automatically determines if it was the result of the capture of
+ * a single stream, or if was the result of certain qualifying union operations. If this is the case the sketch will
+ * take advantage of Edith Cohen's Historical Inverse Probability (HIP) estimation algorithm[2], which was
+ * also independently developed by Daniel Ting as the Martingale estimation algorithm[3].
+ * This will result in a 20% improvement in accuracy over the standard Flajolet estimator.
+ * If it is not a single stream or if the specific union operation did not qualify,
+ * the estimator will default to the Composite Estimator.
+ *
+ * <h4>Composite Estimator</h4>
+ * This advanced estimator is a blend of several algorithms including new algorithms developed by Kevin Lang for his
+ * Compressed Probabilistic Counting (CPC) sketch[4]. These algorithms provide near optimal estimation accuracy
+ * for cases that don't qualify for HIP / Martingale estimation.
+ *
+ * <p>As a result of all of this work on accuracy, one will get a very smooth curve of the underlying accuracy of the
+ * sketch once the statistical randomness is removed through multiple trials. This can be observed in the
+ * following graph.</p>
+ *
+ * <p><img src="doc-files/HLL_HIP_K12T20U20.png" width="500" alt="HLL Accuracy">[6]</p>
+ *
+ * <p>The above graph has 7 curves. At y = 0, is the median line that hugs the x-axis so closely that it can't be seen.
+ * The two curves, just above and just below the x-axis, correspond to +/- 1 standard deviation (SD) of error.
+ * The distance between either one of this pair and the x-axis is also known as the Relative Standard Error (RSE).
+ * This type of graph for illustrating sketch error we call a "pitchfork plot".</p>
+ *
+ * <p>The next two curves above and below correspond to +/- 2 SD, and
+ * the top-most and bottom-most curves correspond to +/- 3 SD.
+ * The chart grid lines are set at +/- multiples of Relative Standard Error (RSE) that correspond to +/- 1,2,3 SD.
+ * Below the cardinality of about 512 there is no error at all. This is the point where this particular
+ * sketch transitions from sparse to dense (or estimation) mode.</p>
+ *
+ * <h3>Three HLL Types</h3>
+ * This HLL implementation offers three different types of HLL sketch, each with different
+ * trade-offs with accuracy, space and performance. These types are selected with the
+ * {@link org.apache.datasketches.hll.TgtHllType TgtHllType} parameter.
+ *
+ * <p>In terms of accuracy, all three types, for the same <i>lgConfigK</i>, have the same error
+ * distribution as a function of cardinality.</p>
+ *
+ * <p>The configuration parameter <i>lgConfigK</i> is the log-base-2 of <i>K</i>,
+ * where <i>K</i> is the number of buckets or slots for the sketch. <i>lgConfigK</i> impacts both accuracy and
+ * the size of the sketch in memory and when stored.</p>
+ *
+ * <h4>HLL 8</h4>
+ * This uses an 8-bit byte per HLL bucket. It is generally the
+ * fastest in terms of update time but has the largest storage footprint of about <i>K</i> bytes.
+ *
+ * <h4>HLL 6</h4>
+ * This uses a 6-bit field per HLL bucket. It is the generally the next fastest
+ * in terms of update time with a storage footprint of about <i>3/4 * K</i> bytes.
+ *
+ * <h4>HLL 4</h4>
+ * This uses a 4-bit field per HLL bucket and for large counts may require
+ * the use of a small internal auxiliary array for storing statistical exceptions, which are rare.
+ * For the values of <i>lgConfigK > 13</i> (<i>K</i> = 8192),
+ * this additional array adds about 3% to the overall storage. It is generally the slowest in
+ * terms of update time, but has the smallest storage footprint of about <i>K/2 * 1.03</i> bytes.
+ *
+ * <h3>Off-Heap Operation</h3>
+ * This HLL sketch also offers the capability of operating off-heap. Given a <i>WritableMemory[5]</i> object
+ * created by the user, the sketch will perform all of its updates and internal phase transitions
+ * in that object, which can actually reside either on-heap or off-heap based on how it was
+ * configured. In large systems that must update and union many millions of sketches, having the
+ * sketch operate off-heap avoids the serialization and deserialization costs of moving sketches from heap to
+ * off-heap and back, and reduces the need for garbage collection.
+ *
+ * <h3>Merging sketches with different configured <i>lgConfigK</i></h3>
+ * This enables a user to union a HLL sketch that was configured with, say, <i>lgConfigK = 12</i>
+ * with another loaded HLL sketch that was configured with, say, <i>lgConfigK = 14</i>.
+ *
+ * <p>Why is this important? Suppose you have been building a history of sketches of your customer's
+ * data that go back a full year (or 5 or 10!) that were all configured with <i>lgConfigK = 12</i>. Because sketches
+ * are so much smaller than the raw data it is possible that the raw data was discarded keeping only the sketches.
+ * Even if you have the raw data, it might be very expensive and time consuming to reload and rebuild all your
+ * sketches with a larger more accurate size, say, <i>lgConfigK = 14</i>.
+ * This capability enables you to merge last year's data with this year's data built with larger sketches and still
+ * have meaningful results.</p>
+ *
+ * <p>In other words, you can change your mind about what size sketch you need for your application at any time and
+ * will not lose access to the data contained in your older historical sketches.</p>
+ *
+ * <p>This capability does come with a caveat: The resulting accuracy of the merged sketch will be the accuracy of the
+ * smaller of the two sketches. Without this capability, you would either be stuck with the configuration you first
+ * chose forever, or you would have to rebuild all your sketches from scratch, or worse, not be able to recover your
+ * historical data.</p>
+ *
+ * <h3>Multi-language, multi-platform.</h3>
+ * The binary structures for our sketch serializations are language and platform independent.
+ * This means it is possible to generate an HLL sketch on a C++ Windows platform and it can be used on a
+ * Java or Python Unix platform.
+ *
+ * <p>[1] Philippe Flajolet, et al, <a href="https://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf">
+<i>HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm.</i></a>
+ * DMTCS proc. <b>AH</b>, 2007, 127-146.
+ *
+ * <p>[2] Edith Cohen, <a href="https://arxiv.org/pdf/1306.3284.pdf">
+<i>All-Distances Sketches, Revisited: HIP Estimators for Massive Graphs Analysis.</i></a>
+ * PODS'14, June 22-27, Snowbird, UT, USA.
+ *
+ * <p>[3] Daniel Ting,
+ * <a href="https://research.facebook.com/publications/streamed-approximate-counting-of-distinct-elements">
+<i>Streamed Approximate Counting of Distinct Elements, Beating Optimal Batch Methods.</i></a>
+ * KDD'14 August 24, 2014 New York, New York USA.
+ *
+ * <p>[4] Kevin Lang,
+ * <a href="https://arxiv.org/abs/1708.06839">
+<i>Back to the Future: an Even More Nearly Optimal Cardinality Estimation Algorithm.</i></a>
+ * arXiv 1708.06839, August 22, 2017, Yahoo Research.
+ *
+ * <p>[5] Memory Component,
+ * <a href="https://datasketches.apache.org/docs/Memory/MemoryComponent.html">
+<i>DataSketches Memory Component</i></a>
+ *
+ * <p>[6] MacBook Pro 2.3 GHz 8-Core Intel Core i9
+ *
+ * @see org.apache.datasketches.cpc.CpcSketch
+ *
+ * @author Lee Rhodes
+ * @author Kevin Lang
+ */
+package org.apache.datasketches.hll2;
diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
index 07904f0ad..08ec155d7 100644
--- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
+++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java
@@ -42,7 +42,7 @@ final class AnotBimpl extends AnotB {
private final short seedHash_;
private boolean empty_;
private long thetaLong_;
- private long[] hashArr_ = new long[0]; //compact array w curCount_ entries
+ private long[] hashArr_ = {}; //compact array w curCount_ entries
private int curCount_;
/**
@@ -86,7 +86,7 @@ final class AnotBimpl extends AnotB {
@Override
public void notB(final Sketch skB) {
- if (empty_ || skB == null || skB.isEmpty()) { return; }
+ if (empty_ || (skB == null) || skB.isEmpty()) { return; }
//local and skB is not empty
Util.checkSeedHashes(seedHash_, skB.getSeedHash());
@@ -95,7 +95,7 @@ final class AnotBimpl extends AnotB {
//process B
hashArr_ = getResultHashArr(thetaLong_, curCount_, hashArr_, skB);
curCount_ = hashArr_.length;
- empty_ = curCount_ == 0 && thetaLong_ == Long.MAX_VALUE;
+ empty_ = (curCount_ == 0) && (thetaLong_ == Long.MAX_VALUE);
}
@Override
@@ -115,7 +115,7 @@ final class AnotBimpl extends AnotB {
@Override
public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dstOrdered,
final MemorySegment dstSeg) {
- if (skA == null || skB == null) {
+ if ((skA == null) || (skB == null)) {
throw new SketchesArgumentException("Neither argument may be null");
}
//Both skA & skB are not null
@@ -139,11 +139,10 @@ final class AnotBimpl extends AnotB {
//process B
final long[] hashArrOut = getResultHashArr(minThetaLong, countA, hashArrA, skB); //out is clone
final int countOut = hashArrOut.length;
- final boolean empty = countOut == 0 && minThetaLong == Long.MAX_VALUE;
+ final boolean empty = (countOut == 0) && (minThetaLong == Long.MAX_VALUE);
- final CompactSketch result = CompactOperations.componentsToCompact(
+ return CompactOperations.componentsToCompact(
minThetaLong, countOut, seedHash_, empty, true, false, dstOrdered, dstSeg, hashArrOut);
- return result;
}
@Override
@@ -156,8 +155,7 @@ final class AnotBimpl extends AnotB {
private static long[] getHashArrA(final Sketch skA) { //returns a new array
//Get skA cache as array
final CompactSketch cskA = skA.compact(false, null); //sorting not required
- final long[] hashArrA = cskA.getCache().clone();
- return hashArrA;
+ return cskA.getCache().clone();
}
private static long[] getResultHashArr( //returns a new array
@@ -182,7 +180,7 @@ final class AnotBimpl extends AnotB {
int nonMatches = 0;
for (int i = 0; i < countA; i++) {
final long hash = hashArrA[i];
- if (hash != 0 && hash < minThetaLong) { //only allows hashes of A < minTheta
+ if ((hash != 0) && (hash < minThetaLong)) { //only allows hashes of A < minTheta
final int index = hashSearch(hashTableB, lgHTBLen, hash);
if (index == -1) {
tmpHashArrA[nonMatches] = hash;
@@ -238,7 +236,7 @@ final class AnotBimpl extends AnotB {
public boolean hasMemorySegment() { return false; }
@Override
- public boolean isDirect() { return false; }
+ public boolean isOffHeap() { return false; }
@Override
public boolean isSameResource( final MemorySegment that) { return false; }
diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
index 413023042..2d8f22a09 100644
--- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java
@@ -344,7 +344,7 @@ public abstract class CompactSketch extends Sketch {
@Override
public boolean hasMemorySegment() {
- return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).hasMemorySegment());
+ return ((this instanceof DirectCompactSketch) && ((DirectCompactSketch)this).hasMemorySegment());
}
@Override
@@ -353,13 +353,13 @@ public abstract class CompactSketch extends Sketch {
}
@Override
- public boolean isDirect() {
- return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).isDirect());
+ public boolean isOffHeap() {
+ return ((this instanceof DirectCompactSketch) && ((DirectCompactSketch)this).isOffHeap());
}
@Override
public boolean isSameResource(final MemorySegment that) {
- return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).isSameResource(that));
+ return ((this instanceof DirectCompactSketch) && ((DirectCompactSketch)this).isSameResource(that));
}
@Override
@@ -372,7 +372,7 @@ public abstract class CompactSketch extends Sketch {
* @return the sketch as a compressed byte array
*/
public byte[] toByteArrayCompressed() {
- if (!isOrdered() || getRetainedEntries() == 0 || (getRetainedEntries() == 1 && !isEstimationMode())) {
+ if (!isOrdered() || (getRetainedEntries() == 0) || ((getRetainedEntries() == 1) && !isEstimationMode())) {
return toByteArray();
}
return toByteArrayV4();
@@ -400,7 +400,7 @@ public abstract class CompactSketch extends Sketch {
// store num_entries as whole bytes since whole-byte blocks will follow (most probably)
final int numEntriesBytes = wholeBytesToHoldBits(32 - Integer.numberOfLeadingZeros(getRetainedEntries()));
- final int sizeBytes = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits);
+ final int sizeBytes = (preambleLongs * Long.BYTES) + numEntriesBytes + wholeBytesToHoldBits(compressedBits);
final byte[] bytes = new byte[sizeBytes];
final MemorySegment wseg = MemorySegment.ofArray(bytes);
int offsetBytes = 0;
@@ -425,7 +425,7 @@ public abstract class CompactSketch extends Sketch {
final long[] deltas = new long[8];
final HashIterator it = iterator();
int i;
- for (i = 0; i + 7 < getRetainedEntries(); i += 8) {
+ for (i = 0; (i + 7) < getRetainedEntries(); i += 8) {
for (int j = 0; j < 8; j++) {
it.next();
deltas[j] = it.get() - previous;
@@ -465,7 +465,7 @@ public abstract class CompactSketch extends Sketch {
final long[] entries = new long[numEntries];
final byte[] bytes = new byte[entryBits]; // temporary buffer for unpacking
int i;
- for (i = 0; i + 7 < numEntries; i += 8) {
+ for (i = 0; (i + 7) < numEntries; i += 8) {
MemorySegment.copy(srcSeg, JAVA_BYTE, offsetBytes, bytes, 0, entryBits);
BitPacking.unpackBitsBlock8(entries, i, bytes, 0, entryBits);
offsetBytes += entryBits;
diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java
index 5158e1c80..184b34ebe 100644
--- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java
+++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java
@@ -154,8 +154,8 @@ final class ConcurrentHeapThetaBuffer extends HeapQuickSelectSketch {
}
@Override
- public boolean isDirect() {
- return shared.isDirect();
+ public boolean isOffHeap() {
+ return shared.isOffHeap();
}
@Override
diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
index 693329110..2fcbf08d6 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java
@@ -33,6 +33,7 @@ import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingle
import java.lang.foreign.MemorySegment;
+import org.apache.datasketches.common.MemorySegmentStatus;
import org.apache.datasketches.common.Util;
/**
@@ -90,8 +91,7 @@ class DirectCompactSketch extends CompactSketch {
public int getRetainedEntries(final boolean valid) { //compact is always valid
if (otherCheckForSingleItem(seg_)) { return 1; }
final int preLongs = extractPreLongs(seg_);
- final int curCount = (preLongs == 1) ? 0 : extractCurCount(seg_);
- return curCount;
+ return (preLongs == 1) ? 0 : extractCurCount(seg_);
}
@Override
@@ -102,11 +102,11 @@ class DirectCompactSketch extends CompactSketch {
@Override
public boolean hasMemorySegment() {
- return seg_ != null && seg_.scope().isAlive();
+ return (seg_ != null) && seg_.scope().isAlive();
}
@Override
- public boolean isDirect() {
+ public boolean isOffHeap() {
return hasMemorySegment() && seg_.isNative();
}
@@ -125,7 +125,7 @@ class DirectCompactSketch extends CompactSketch {
@Override
public boolean isSameResource(final MemorySegment that) {
- return hasMemorySegment() && Util.isSameResource(seg_, that);
+ return hasMemorySegment() && MemorySegmentStatus.isSameResource(seg_, that);
}
diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java
index 2c25ee618..0a81f4887 100644
--- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java
+++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java
@@ -44,10 +44,10 @@ import static org.apache.datasketches.theta.PreambleUtil.insertThetaLong;
import java.lang.foreign.MemorySegment;
import org.apache.datasketches.common.Family;
+import org.apache.datasketches.common.MemorySegmentStatus;
import org.apache.datasketches.common.ResizeFactor;
import org.apache.datasketches.common.SketchesReadOnlyException;
import org.apache.datasketches.common.SuppressFBWarnings;
-import org.apache.datasketches.common.Util;
import org.apache.datasketches.thetacommon.ThetaUtil;
/**
@@ -118,8 +118,7 @@ class DirectQuickSelectSketchR extends UpdateSketch {
//not compact
final byte lgArrLongs = wseg_.get(JAVA_BYTE, LG_ARR_LONGS_BYTE);
final int preLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F;
- final int lengthBytes = (preLongs + (1 << lgArrLongs)) << 3;
- return lengthBytes;
+ return (preLongs + (1 << lgArrLongs)) << 3;
}
@Override
@@ -147,11 +146,11 @@ class DirectQuickSelectSketchR extends UpdateSketch {
@Override
public boolean hasMemorySegment() {
- return wseg_ != null && wseg_.scope().isAlive();
+ return (wseg_ != null) && wseg_.scope().isAlive();
}
@Override
- public boolean isDirect() {
+ public boolean isOffHeap() {
return hasMemorySegment() && wseg_.isNative();
}
@@ -162,7 +161,7 @@ class DirectQuickSelectSketchR extends UpdateSketch {
@Override
public boolean isSameResource(final MemorySegment that) {
- return hasMemorySegment() && Util.isSameResource(wseg_, that);
+ return hasMemorySegment() && MemorySegmentStatus.isSameResource(wseg_, that);
}
@Override
diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
index 6bf922be0..6db8f6da1 100644
--- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java
@@ -62,6 +62,7 @@ import java.lang.foreign.MemorySegment;
import java.util.Arrays;
import org.apache.datasketches.common.Family;
+import org.apache.datasketches.common.MemorySegmentStatus;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesReadOnlyException;
import org.apache.datasketches.common.SketchesStateException;
@@ -186,12 +187,10 @@ final class IntersectionImpl extends Intersection {
impl.curCount_ = extractCurCount(srcSeg);
impl.thetaLong_ = extractThetaLong(srcSeg);
impl.empty_ = (extractFlags(srcSeg) & EMPTY_FLAG_MASK) > 0;
- if (!impl.empty_) {
- if (impl.curCount_ > 0) {
+ if (!impl.empty_ && (impl.curCount_ > 0)) {
impl.hashTable_ = new long[1 << impl.lgArrLongs_];
MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, CONST_PREAMBLE_LONGS << 3, impl.hashTable_, 0, 1 << impl.lgArrLongs_);
}
- }
return impl;
}
@@ -219,7 +218,7 @@ final class IntersectionImpl extends Intersection {
@Override
public CompactSketch intersect(final Sketch a, final Sketch b, final boolean dstOrdered, final MemorySegment dstSeg) {
- if (wseg_ != null && readOnly_) { throw new SketchesReadOnlyException(); }
+ if ((wseg_ != null) && readOnly_) { throw new SketchesReadOnlyException(); }
hardReset();
intersect(a);
intersect(b);
@@ -233,7 +232,7 @@ final class IntersectionImpl extends Intersection {
if (sketchIn == null) {
throw new SketchesArgumentException("Intersection argument must not be null.");
}
- if (wseg_ != null && readOnly_) { throw new SketchesReadOnlyException(); }
+ if ((wseg_ != null) && readOnly_) { throw new SketchesReadOnlyException(); }
if (empty_ || sketchIn.isEmpty()) { //empty rule
//Because of the def of null above and the Empty Rule (which is OR), empty_ must be true.
//Whatever the current internal state, we make our local empty.
@@ -262,14 +261,14 @@ final class IntersectionImpl extends Intersection {
final int sketchInEntries = sketchIn.getRetainedEntries(true);
//states 1,2,3,6
- if (curCount_ == 0 || sketchInEntries == 0) {
+ if ((curCount_ == 0) || (sketchInEntries == 0)) {
curCount_ = 0;
if (wseg_ != null) { insertCurCount(wseg_, 0); }
hashTable_ = null; //No need for a HT. Don't bother clearing seg if valid
} //end of states 1,2,3,6
// state 5
- else if (curCount_ < 0 && sketchInEntries > 0) {
+ else if ((curCount_ < 0) && (sketchInEntries > 0)) {
curCount_ = sketchIn.getRetainedEntries(true);
final int requiredLgArrLongs = minLgHashTableSize(curCount_, ThetaUtil.REBUILD_THRESHOLD);
final int priorLgArrLongs = lgArrLongs_; //prior only used in error message
@@ -295,7 +294,7 @@ final class IntersectionImpl extends Intersection {
} //end of state 5
//state 7
- else if (curCount_ > 0 && sketchInEntries > 0) {
+ else if ((curCount_ > 0) && (sketchInEntries > 0)) {
//Sets resulting hashTable, curCount and adjusts lgArrLongs
performIntersect(sketchIn);
} //end of state 7
@@ -344,7 +343,7 @@ final class IntersectionImpl extends Intersection {
@Override
public boolean hasMemorySegment() {
- return wseg_ != null && wseg_.scope().isAlive();
+ return (wseg_ != null) && wseg_.scope().isAlive();
}
@Override
@@ -353,13 +352,13 @@ final class IntersectionImpl extends Intersection {
}
@Override
- public boolean isDirect() {
+ public boolean isOffHeap() {
return hasMemorySegment() && wseg_.isNative();
}
@Override
public boolean isSameResource(final MemorySegment that) {
- return hasMemorySegment() && Util.isSameResource(wseg_, that);
+ return hasMemorySegment() && MemorySegmentStatus.isSameResource(wseg_, that);
}
@Override
@@ -439,7 +438,7 @@ final class IntersectionImpl extends Intersection {
private void performIntersect(final Sketch sketchIn) {
// curCount and input data are nonzero, match against HT
- assert curCount_ > 0 && !empty_;
+ assert (curCount_ > 0) && !empty_;
final long[] hashTable;
if (wseg_ != null) {
final int htLen = 1 << lgArrLongs_;
@@ -461,9 +460,7 @@ final class IntersectionImpl extends Intersection {
if (foundIdx != -1) {
matchSet[matchSetCount++] = hashIn;
}
- } else {
- if (isOrdered) { break; } // early stop
- }
+ } else if (isOrdered) { break; } // early stop
}
//reduce effective array size to minimum
curCount_ = matchSetCount;
@@ -478,11 +475,9 @@ final class IntersectionImpl extends Intersection {
if (curCount_ > 0) {
moveDataToTgt(matchSet, matchSetCount); //move matchSet to target
- } else {
- if (thetaLong_ == Long.MAX_VALUE) {
+ } else if (thetaLong_ == Long.MAX_VALUE) {
empty_ = true;
}
- }
}
private void moveDataToTgt(final long[] arr, final int count) {
diff --git a/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java
index 031cee687..5a474e49e 100644
--- a/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java
+++ b/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java
@@ -25,21 +25,20 @@ import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
import java.lang.foreign.MemorySegment;
import org.apache.datasketches.common.MemorySegmentStatus;
-import org.apache.datasketches.common.Util;
/*
* This is to uncompress serial version 4 sketch incrementally
*/
final class MemorySegmentCompactCompressedHashIterator implements HashIterator, MemorySegmentStatus {
- private MemorySegment seg;
+ private final MemorySegment seg;
private int offset;
- private int entryBits;
- private int numEntries;
+ private final int entryBits;
+ private final int numEntries;
private int index;
private long previous;
private int offsetBits;
- private long[] buffer;
- private byte[] bytes;
+ private final long[] buffer;
+ private final byte[] bytes;
private boolean isBlockMode;
private boolean isFirstUnpack1;
@@ -48,7 +47,7 @@ final class MemorySegmentCompactCompressedHashIterator implements HashIterator,
final int offset,
final int entryBits,
final int numEntries) {
- this.seg = srcSeg;
+ seg = srcSeg;
this.offset = offset;
this.entryBits = entryBits;
this.numEntries = numEntries;
@@ -68,17 +67,17 @@ final class MemorySegmentCompactCompressedHashIterator implements HashIterator,
@Override
public boolean hasMemorySegment() {
- return seg != null && seg.scope().isAlive();
+ return (seg != null) && seg.scope().isAlive();
}
@Override
- public boolean isDirect() {
+ public boolean isOffHeap() {
return hasMemorySegment() && seg.isNative();
}
@Override
public boolean isSameResource(final MemorySegment that) {
- return hasMemorySegment() && Util.isSameResource(seg, that);
+ return hasMemorySegment() && MemorySegmentStatus.isSameResource(seg, that);
}
@Override
@@ -86,7 +85,7 @@ final class MemorySegmentCompactCompressedHashIterator implements HashIterator,
if (++index == numEntries) { return false; }
if (isBlockMode) {
if ((index & 7) == 0) {
- if (numEntries - index >= 8) {
+ if ((numEntries - index) >= 8) {
unpack8();
} else {
isBlockMode = false;
diff --git a/src/main/java/org/apache/datasketches/theta/SetOperation.java b/src/main/java/org/apache/datasketches/theta/SetOperation.java
index 126a9298a..72e7a4960 100644
--- a/src/main/java/org/apache/datasketches/theta/SetOperation.java
+++ b/src/main/java/org/apache/datasketches/theta/SetOperation.java
@@ -166,8 +166,7 @@ public abstract class SetOperation implements MemorySegmentStatus {
*/
public static int getMaxIntersectionBytes(final int nomEntries) {
final int nomEnt = ceilingPowerOf2(nomEntries);
- final int bytes = (nomEnt << 4) + (Family.INTERSECTION.getMaxPreLongs() << 3);
- return bytes;
+ return (nomEnt << 4) + (Family.INTERSECTION.getMaxPreLongs() << 3);
}
/**
@@ -226,12 +225,6 @@ public abstract class SetOperation implements MemorySegmentStatus {
*/
abstract long getThetaLong();
- @Override
- public abstract boolean hasMemorySegment();
-
- @Override
- public abstract boolean isDirect();
-
/**
* Returns true if this set operator is empty.
* Only useful during stateful operations.
@@ -240,7 +233,4 @@ public abstract class SetOperation implements MemorySegmentStatus {
*/
abstract boolean isEmpty();
- @Override
- public abstract boolean isSameResource(final MemorySegment seg);
-
}
diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index 387ee3455..ed0178c8c 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -226,8 +226,8 @@ final class UnionImpl extends Union {
}
@Override
- public boolean isDirect() {
- return gadget_.isDirect();
+ public boolean isOffHeap() {
+ return gadget_.isOffHeap();
}
@Override
@@ -269,7 +269,7 @@ final class UnionImpl extends Union {
public void union(final Sketch sketchIn) {
//UNION Empty Rule: AND the empty states.
- if (sketchIn == null || sketchIn.isEmpty()) {
+ if ((sketchIn == null) || sketchIn.isEmpty()) {
//null and empty is interpreted as (Theta = 1.0, count = 0, empty = T). Nothing changes
return;
}
@@ -287,11 +287,9 @@ final class UnionImpl extends Union {
final HashIterator it = sketchIn.iterator();
while (it.next()) {
final long hash = it.get();
- if (hash < unionThetaLong_ && hash < gadget_.getThetaLong()) {
+ if ((hash < unionThetaLong_) && (hash < gadget_.getThetaLong())) {
gadget_.hashUpdate(hash); // backdoor update, hash function is bypassed
- } else {
- if (isOrdered) { break; }
- }
+ } else if (isOrdered) { break; }
}
unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //Theta rule with gadget
if (gadget_.hasMemorySegment()) {
diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
index 30d3b6f49..e5aa3d405 100644
--- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
@@ -161,7 +161,7 @@ public abstract class UpdateSketch extends Sketch {
@Override
public boolean hasMemorySegment() {
- return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).hasMemorySegment());
+ return ((this instanceof DirectQuickSelectSketchR) && ((DirectQuickSelectSketchR)this).hasMemorySegment());
}
@Override
@@ -170,8 +170,8 @@ public abstract class UpdateSketch extends Sketch {
}
@Override
- public boolean isDirect() {
- return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).isDirect());
+ public boolean isOffHeap() {
+ return ((this instanceof DirectQuickSelectSketchR) && ((DirectQuickSelectSketchR)this).isOffHeap());
}
@Override
@@ -181,7 +181,7 @@ public abstract class UpdateSketch extends Sketch {
@Override
public boolean isSameResource(final MemorySegment that) {
- return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).isSameResource(that));
+ return ((this instanceof DirectQuickSelectSketchR) && ((DirectQuickSelectSketchR)this).isSameResource(that));
}
//UpdateSketch interface
@@ -299,7 +299,7 @@ public abstract class UpdateSketch extends Sketch {
* <a href="{@docRoot}/resources/dictionary.html#updateReturnState">See Update Return State</a>
*/
public UpdateReturnState update(final ByteBuffer buffer) {
- if (buffer == null || buffer.hasRemaining() == false) {
+ if ((buffer == null) || !buffer.hasRemaining()) {
return RejectedNullOrEmpty;
}
return hashUpdate(hash(buffer, getSeed())[0] >>> 1);
@@ -479,7 +479,7 @@ public abstract class UpdateSketch extends Sketch {
final int lgA = lgArrLongs;
final int lgR = extractLgResizeFactor(srcSeg);
if (lgR == 0) { return lgA != lgT; }
- return !(((lgT - lgA) % lgR) == 0);
+ return (((lgT - lgA) % lgR) != 0);
}
}
diff --git a/src/test/java/org/apache/datasketches/hll2/AuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll2/AuxHashMapTest.java
new file mode 100644
index 000000000..274bf14fa
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll2/AuxHashMapTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.fail;
+
+import org.apache.datasketches.common.SketchesStateException;
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+public class AuxHashMapTest {
+ static final String LS = System.getProperty("line.separator");
+
+ @Test
+ public void checkMustReplace() {
+ HeapAuxHashMap map = new HeapAuxHashMap(3, 7);
+ map.mustAdd(100, 5);
+ int val = map.mustFindValueFor(100);
+ assertEquals(val, 5);
+
+ map.mustReplace(100, 10);
+ val = map.mustFindValueFor(100);
+ assertEquals(val, 10);
+
+ try {
+ map.mustReplace(101, 5);
+ fail();
+ } catch (SketchesStateException e) {
+ //expected
+ }
+ }
+
+ @Test
+ public void checkGrowSpace() {
+ HeapAuxHashMap map = new HeapAuxHashMap(3, 7);
+ assertFalse(map.hasMemorySegment());
+ assertFalse(map.isOffHeap());
+ assertEquals(map.getLgAuxArrInts(), 3);
+ for (int i = 1; i <= 7; i++) {
+ map.mustAdd(i, i);
+ }
+ assertEquals(map.getLgAuxArrInts(), 4);
+ PairIterator itr = map.getIterator();
+ int count1 = 0;
+ int count2 = 0;
+ while (itr.nextAll()) {
+ count2++;
+ int pair = itr.getPair();
+ if (pair != 0) { count1++; }
+ }
+ assertEquals(count1, 7);
+ assertEquals(count2, 16);
+ }
+
+ @Test(expectedExceptions = SketchesStateException.class)
+ public void checkExceptions1() {
+ HeapAuxHashMap map = new HeapAuxHashMap(3, 7);
+ map.mustAdd(100, 5);
+ map.mustFindValueFor(101);
+ }
+
+ @Test(expectedExceptions = SketchesStateException.class)
+ public void checkExceptions2() {
+ HeapAuxHashMap map = new HeapAuxHashMap(3, 7);
+ map.mustAdd(100, 5);
+ map.mustAdd(100, 6);
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: " + this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/hll2/BaseHllSketchTest.java b/src/test/java/org/apache/datasketches/hll2/BaseHllSketchTest.java
new file mode 100644
index 000000000..95e1dead4
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll2/BaseHllSketchTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import org.testng.annotations.Test;
+
+import java.lang.foreign.MemorySegment;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @author Lee Rhodes
+ *
+ */
+public class BaseHllSketchTest {
+
+ @Test
+ public void checkUpdateTypes() {
+ final HllSketch sk = new HllSketch(10);
+ final byte[] byteArr = null;
+ sk.update(byteArr);
+ sk.update(new byte[] {});
+ sk.update(new byte[] {0, 1, 2, 3});
+ final ByteBuffer byteBuf = null;
+ sk.update(byteBuf);
+ sk.update(ByteBuffer.wrap(new byte[] {}));
+ sk.update(ByteBuffer.wrap(new byte[] {0, 1, 2, 3}));
+ final char[] charArr = null;
+ sk.update(charArr);
+ sk.update(new char[] {});
+ sk.update(new char[] {0, 1, 2, 3});
+ sk.update(1.0);
+ sk.update(-0.0);
+ final int[] intArr = null;
+ sk.update(intArr);
+ sk.update(new int[] {});
+ sk.update(new int[] {0, 1, 2, 3});
+ sk.update(1234L);
+ final long[] longArr = null;
+ sk.update(longArr);
+ sk.update(new long[] {});
+ sk.update(new long[] {0, 1, 2, 3});
+ String s = null;
+ sk.update(s);
+ s = "";
+ sk.update(s);
+ sk.update("1234");
+
+ final Union u = new Union(10);
+ final byte[] byteArr1 = null;
+ u.update(byteArr1);
+ u.update(new byte[] {});
+ u.update(new byte[] {0, 1, 2, 3});
+ final ByteBuffer byteBuf1 = null;
+ u.update(byteBuf1);
+ u.update(ByteBuffer.wrap(new byte[] {}));
+ u.update(ByteBuffer.wrap(new byte[] {0, 1, 2, 3}));
+ final char[] charArr1 = null;
+ u.update(charArr1);
+ u.update(new char[] {});
+ u.update(new char[] {0, 1, 2, 3});
+ u.update(1.0);
+ u.update(-0.0);
+ final int[] intArr1 = null;
+ u.update(intArr1);
+ u.update(new int[] {});
+ u.update(new int[] {0, 1, 2, 3});
+ u.update(1234L);
+ final long[] longArr1 = null;
+ u.update(longArr1);
+ u.update(new long[] {});
+ u.update(new long[] {0, 1, 2, 3});
+ String s1 = null;
+ u.update(s1);
+ s1 = "";
+ u.update(s);
+ u.update("1234");
+ }
+
+ @Test
+ public void misc() {
+ final HllSketch sk = new HllSketch(10, TgtHllType.HLL_4);
+ assertTrue(sk.isEstimationMode());
+ sk.reset();
+ assertEquals(BaseHllSketch.getSerializationVersion(), PreambleUtil.SER_VER);
+ final MemorySegment wseg = MemorySegment.ofArray(sk.toCompactByteArray());
+ assertEquals(BaseHllSketch.getSerializationVersion(wseg), PreambleUtil.SER_VER);
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: " + this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(final String s) {
+ //System.out.println(s); //disable here
+ }
+}
diff --git a/src/test/java/org/apache/datasketches/hll2/CouponListTest.java b/src/test/java/org/apache/datasketches/hll2/CouponListTest.java
new file mode 100644
index 000000000..c27f37c52
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll2/CouponListTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll2;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import java.lang.foreign.MemorySegment;
+
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+public class CouponListTest {
+
+ @Test //visual check
+ public void checkIterator() {
+ checkIterator(false, 4, 7);
+ checkIterator(true, 4, 7);
+ }
+
+ private static void checkIterator(final boolean direct, final int lgK, final int n) {
+ final TgtHllType tgtHllType = TgtHllType.HLL_4;
+ final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+ final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]);
+ final HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wseg) : new HllSketch(lgK, tgtHllType);
+ for (int i = 0; i < n; i++) { sk.update(i); }
+ final String store = direct ? "MemorySegment" : "Heap";
+ println("CurMode: " + sk.getCurMode().toString() + "; Store: " + store);
+ final PairIterator itr = sk.iterator();
+ println(itr.getHeader());
+ while (itr.nextAll()) {
+ assertTrue(itr.getSlot() < (1 << lgK));
+ println(itr.getString());
+ }
+ }
+
+ @Test
+ public void checkDuplicatesAndMisc() {
+ checkDuplicatesAndMisc(false);
+ checkDuplicatesAndMisc(true);
+ }
+
+ private static void checkDuplicatesAndMisc(final boolean direct) {
... 9866 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org