You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by jm...@apache.org on 2024/01/25 05:24:25 UTC
(datasketches-java) 01/02: Update family and sampling preamble utils to support EBPPS
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch ebpps
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 2f5ff7a3e9623ff89b23fe496d6d20da1232d9c8
Author: jmalkin <78...@users.noreply.github.com>
AuthorDate: Wed Jan 24 21:23:48 2024 -0800
Update family and sampling preamble utils to support EBPPS
---
.../org/apache/datasketches/common/Family.java | 7 +++-
.../apache/datasketches/sampling/PreambleUtil.java | 44 +++++++++++++++++++++-
.../sampling/ReservoirItemsSketch.java | 8 ++--
.../datasketches/sampling/ReservoirItemsUnion.java | 8 ++--
.../sampling/ReservoirLongsSketch.java | 8 ++--
.../datasketches/sampling/ReservoirLongsUnion.java | 8 ++--
.../datasketches/sampling/VarOptItemsSketch.java | 8 ++--
.../datasketches/sampling/VarOptItemsUnion.java | 8 ++--
8 files changed, 73 insertions(+), 26 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/common/Family.java b/src/main/java/org/apache/datasketches/common/Family.java
index cac7722e..cac049e9 100644
--- a/src/main/java/org/apache/datasketches/common/Family.java
+++ b/src/main/java/org/apache/datasketches/common/Family.java
@@ -146,7 +146,12 @@ public enum Family {
/**
* CountMin Sketch
*/
- COUNTMIN(18, "COUNTMIN", 2, 2);
+ COUNTMIN(18, "COUNTMIN", 2, 2),
+
+ /**
+ * Exact and Bounded, Probability Proportional to Size (EBPPS)
+ */
+ EBPPS(19, "EBPPS", 1, 5);
private static final Map<Integer, Family> lookupID = new HashMap<>();
private static final Map<String, Family> lookupFamName = new HashMap<>();
diff --git a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java
index 453e3901..9bc5ac8b 100644
--- a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java
@@ -155,6 +155,14 @@ final class PreambleUtil {
static final int VO_PRELONGS_WARMUP = 3; // Doesn't match min or max prelongs in Family
static final int VO_PRELONGS_FULL = Family.VAROPT.getMaxPreLongs();
+ // constants and addresses used in EBPPS
+ static final int EBPPS_CUM_WT_DOUBLE = 16;
+ static final int EBPPS_MAX_WT_DOUBLE = 24;
+ static final int EBPPS_RHO_DOUBLE = 32;
+ static final int EBPPS_C_DOUBLE = 40;
+ static final int EBPPS_ITEMS_START = 48;
+
+
// flag bit masks
//static final int BIG_ENDIAN_FLAG_MASK = 1;
//static final int READ_ONLY_FLAG_MASK = 2;
@@ -162,7 +170,9 @@ final class PreambleUtil {
static final int GADGET_FLAG_MASK = 128;
//Other constants
- static final int SER_VER = 2;
+ static final int RESERVOIR_SER_VER = 2;
+ static final int VAROPT_SER_VER = 2;
+ static final int EBPPS_SER_VER = 1;
static final boolean NATIVE_ORDER_IS_BIG_ENDIAN =
(ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN);
@@ -378,6 +388,22 @@ final class PreambleUtil {
return mem.getLong(OUTER_TAU_DENOM_LONG);
}
+ static double extractEbppsCumulativeWeight(final Memory mem) {
+ return mem.getDouble(EBPPS_CUM_WT_DOUBLE);
+ }
+
+ static double extractEbppsMaxWeight(final Memory mem) {
+ return mem.getDouble(EBPPS_MAX_WT_DOUBLE);
+ }
+
+ static double extractEbppsRho(final Memory mem) {
+ return mem.getDouble(EBPPS_RHO_DOUBLE);
+ }
+
+ static double extractEbppsC(final Memory mem) {
+ return mem.getDouble(EBPPS_C_DOUBLE);
+ }
+
// Insertion methods
static void insertPreLongs(final WritableMemory wmem, final int preLongs) {
@@ -439,6 +465,22 @@ final class PreambleUtil {
wmem.putLong(OUTER_TAU_DENOM_LONG, denom);
}
+ static void insertEbppsCumulativeWeight(final WritableMemory wmem, final double cumWt) {
+ wmem.putDouble(EBPPS_CUM_WT_DOUBLE, cumWt);
+ }
+
+ static void insertEbppsMaxWeight(final WritableMemory wmem, final double maxWt) {
+ wmem.putDouble(EBPPS_MAX_WT_DOUBLE, maxWt);
+ }
+
+ static void insertEbppsRho(final WritableMemory wmem, final double rho) {
+ wmem.putDouble(EBPPS_RHO_DOUBLE, rho);
+ }
+
+ static void insertEbppsC(final WritableMemory wmem, final double c) {
+ wmem.putDouble(EBPPS_C_DOUBLE, c);
+ }
+
/**
* Checks Memory for capacity to hold the preamble and returns the extracted preLongs.
* @param mem the given Memory
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java
index dc21d5ef..030728fe 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsSketch.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.FAMILY_BYTE;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.RESERVOIR_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.extractEncodedReservoirSize;
import static org.apache.datasketches.sampling.PreambleUtil.extractFlags;
import static org.apache.datasketches.sampling.PreambleUtil.extractK;
@@ -229,13 +229,13 @@ public final class ReservoirItemsSketch<T> {
+ Family.RESERVOIR.getMinPreLongs() + " preLong(s)");
}
- if (serVer != SER_VER) {
+ if (serVer != RESERVOIR_SER_VER) {
if (serVer == 1) {
final short encK = extractEncodedReservoirSize(srcMem);
k = ReservoirSize.decodeValue(encK);
} else {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + RESERVOIR_SER_VER + ": " + serVer);
}
}
@@ -479,7 +479,7 @@ public final class ReservoirItemsSketch<T> {
// Common header elements
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
PreambleUtil.insertLgResizeFactor(mem, rf_.lg());
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, RESERVOIR_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.RESERVOIR.getID()); // Byte 2
if (empty) {
PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK); // Byte 3
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsUnion.java b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsUnion.java
index 9f4bd5f3..74a52e49 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirItemsUnion.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirItemsUnion.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.FAMILY_BYTE;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.RESERVOIR_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.extractEncodedReservoirSize;
import static org.apache.datasketches.sampling.PreambleUtil.extractFlags;
import static org.apache.datasketches.sampling.PreambleUtil.extractMaxK;
@@ -109,13 +109,13 @@ public final class ReservoirItemsUnion<T> {
+ Family.RESERVOIR_UNION.getMinPreLongs() + "preLongs");
}
- if (serVer != SER_VER) {
+ if (serVer != RESERVOIR_SER_VER) {
if (serVer == 1) {
final short encMaxK = extractEncodedReservoirSize(srcMem);
maxK = ReservoirSize.decodeValue(encMaxK);
} else {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + RESERVOIR_SER_VER + ": " + serVer);
}
}
@@ -304,7 +304,7 @@ public final class ReservoirItemsUnion<T> {
// build preLong
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, RESERVOIR_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.RESERVOIR_UNION.getID()); // Byte 2
if (empty) {
PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK);
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java
index acdfd303..55ca558e 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsSketch.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.FAMILY_BYTE;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.RESERVOIR_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.extractEncodedReservoirSize;
import static org.apache.datasketches.sampling.PreambleUtil.extractFlags;
import static org.apache.datasketches.sampling.PreambleUtil.extractK;
@@ -209,13 +209,13 @@ public final class ReservoirLongsSketch {
+ Family.RESERVOIR.getMinPreLongs() + "preLongs");
}
- if (serVer != SER_VER) {
+ if (serVer != RESERVOIR_SER_VER) {
if (serVer == 1) {
final short encK = extractEncodedReservoirSize(srcMem);
k = ReservoirSize.decodeValue(encK);
} else {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + RESERVOIR_SER_VER + ": " + serVer);
}
}
@@ -407,7 +407,7 @@ public final class ReservoirLongsSketch {
// build first preLong
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
PreambleUtil.insertLgResizeFactor(mem, rf_.lg());
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, RESERVOIR_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.RESERVOIR.getID()); // Byte 2
if (empty) {
PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK); // Byte 3
diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
index f8d9d1eb..c3ef3395 100644
--- a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
+++ b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.FAMILY_BYTE;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.RESERVOIR_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.extractEncodedReservoirSize;
import static org.apache.datasketches.sampling.PreambleUtil.extractFlags;
import static org.apache.datasketches.sampling.PreambleUtil.extractMaxK;
@@ -100,13 +100,13 @@ public final class ReservoirLongsUnion {
+ Family.RESERVOIR_UNION.getMinPreLongs() + "preLongs");
}
- if (serVer != SER_VER) {
+ if (serVer != RESERVOIR_SER_VER) {
if (serVer == 1) {
final short encMaxK = extractEncodedReservoirSize(srcMem);
maxK = ReservoirSize.decodeValue(encMaxK);
} else {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + RESERVOIR_SER_VER + ": " + serVer);
}
}
@@ -257,7 +257,7 @@ public final class ReservoirLongsUnion {
// construct header
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, RESERVOIR_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.RESERVOIR_UNION.getID()); // Byte 2
if (empty) {
PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK); // Byte 3
diff --git a/src/main/java/org/apache/datasketches/sampling/VarOptItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/VarOptItemsSketch.java
index e43ff690..c0c56ba3 100644
--- a/src/main/java/org/apache/datasketches/sampling/VarOptItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/sampling/VarOptItemsSketch.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.GADGET_FLAG_MASK;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.VAROPT_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.TOTAL_WEIGHT_R_DOUBLE;
import static org.apache.datasketches.sampling.PreambleUtil.VO_PRELONGS_EMPTY;
import static org.apache.datasketches.sampling.PreambleUtil.VO_PRELONGS_FULL;
@@ -291,9 +291,9 @@ public final class VarOptItemsSketch<T> {
+ " or " + VO_PRELONGS_FULL + " for a non-empty sketch. Found: " + numPreLongs);
}
}
- if (serVer != SER_VER) {
+ if (serVer != VAROPT_SER_VER) {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + VAROPT_SER_VER + ": " + serVer);
}
final int reqFamilyId = Family.VAROPT.getID();
if (familyId != reqFamilyId) {
@@ -587,7 +587,7 @@ public final class VarOptItemsSketch<T> {
// build first preLong
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
PreambleUtil.insertLgResizeFactor(mem, rf_.lg());
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, VAROPT_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.VAROPT.getID()); // Byte 2
PreambleUtil.insertFlags(mem, flags); // Byte 3
PreambleUtil.insertK(mem, k_); // Bytes 4-7
diff --git a/src/main/java/org/apache/datasketches/sampling/VarOptItemsUnion.java b/src/main/java/org/apache/datasketches/sampling/VarOptItemsUnion.java
index d21c716a..1f1215d7 100644
--- a/src/main/java/org/apache/datasketches/sampling/VarOptItemsUnion.java
+++ b/src/main/java/org/apache/datasketches/sampling/VarOptItemsUnion.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.sampling;
import static org.apache.datasketches.common.Util.LS;
import static org.apache.datasketches.sampling.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.sampling.PreambleUtil.FAMILY_BYTE;
-import static org.apache.datasketches.sampling.PreambleUtil.SER_VER;
+import static org.apache.datasketches.sampling.PreambleUtil.VAROPT_SER_VER;
import static org.apache.datasketches.sampling.PreambleUtil.extractFlags;
import static org.apache.datasketches.sampling.PreambleUtil.extractMaxK;
import static org.apache.datasketches.sampling.PreambleUtil.extractN;
@@ -165,9 +165,9 @@ public final class VarOptItemsUnion<T> {
outerTauDenom = extractOuterTauDenominator(srcMem);
}
- if (serVer != SER_VER) {
+ if (serVer != VAROPT_SER_VER) {
throw new SketchesArgumentException(
- "Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
+ "Possible Corruption: Ser Ver must be " + VAROPT_SER_VER + ": " + serVer);
}
final boolean preLongsEqMin = (numPreLongs == Family.VAROPT_UNION.getMinPreLongs());
@@ -332,7 +332,7 @@ public final class VarOptItemsUnion<T> {
// build preLong
PreambleUtil.insertPreLongs(mem, preLongs); // Byte 0
- PreambleUtil.insertSerVer(mem, SER_VER); // Byte 1
+ PreambleUtil.insertSerVer(mem, VAROPT_SER_VER); // Byte 1
PreambleUtil.insertFamilyID(mem, Family.VAROPT_UNION.getID()); // Byte 2
if (empty) {
PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org