You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2020/04/01 17:35:24 UTC
[incubator-datasketches-characterization] branch HllBoundsTesting
created (now d3c9454)
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a change to branch HllBoundsTesting
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-characterization.git.
at d3c9454 Update Bounds tests
This branch includes the following new commits:
new 8734e15 initial Hll Bounds Testing
new d3c9454 Update Bounds tests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-characterization] 01/02: initial Hll Bounds
Testing
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch HllBoundsTesting
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-characterization.git
commit 8734e15644f40206bd09ec508447dae1a641a4b0
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Tue Mar 31 17:13:45 2020 -0700
initial Hll Bounds Testing
---
pom.xml | 1 +
.../{PerformanceUtil.java => GaussianRanks.java} | 27 +------
.../{TestingUtil2.java => MonotonicPoints.java} | 60 +++++++++-----
.../{ => characterization}/AccuracyStats.java | 20 ++---
.../characterization/BoundsAccuracyStats.java | 94 ++++++++++++++++++++++
.../characterization/fdt/FdtAccuracyProfile.java | 4 +-
.../hll/DruidHllAccuracyProfile.java | 2 +-
.../characterization/hll/HllAccuracyProfile.java | 2 +-
.../hll/HllBoundsAccuracyProfile.java | 88 ++++++++++++++++++++
.../hll/ZetaHllAccuracyProfile.java | 2 +-
.../quantiles/BaseQuantilesAccuracyProfile.java | 4 +-
.../tdigest/QuantilesAccuracyProfile.java | 4 +-
.../theta/ThetaAccuracyProfile.java | 2 +-
.../theta/ThetaUnionAccuracyProfile.java | 2 +-
.../theta/ThetaUnionAccuracyProfile2.java | 2 +-
.../concurrent/ConcurrentThetaAccuracyProfile.java | 2 +-
.../uniquecount/BaseAccuracyProfile.java | 20 ++---
...Profile.java => BaseBoundsAccuracyProfile.java} | 91 ++++++++-------------
.../org/apache/datasketches/hll/HllRedirects.java | 59 ++++++++++++++
.../org/apache/datasketches/hll/package-info.java | 24 ++++++
src/main/resources/hll/HllBoundsAccuracyJob.conf | 47 +++++++++++
21 files changed, 426 insertions(+), 131 deletions(-)
diff --git a/pom.xml b/pom.xml
index aedd827..e59bb35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -341,6 +341,7 @@ under the License.
<exclude>src/main/resources/**/*.conf</exclude>
<exclude>cpp/matlab/**</exclude>
<exclude>cpp/results/**</exclude>
+ <exclude>results/**</exclude>
<exclude>LICENSE</exclude>
<exclude>NOTICE</exclude>
<exclude>DISCLAIMER-WIP</exclude>
diff --git a/src/main/java/org/apache/datasketches/PerformanceUtil.java b/src/main/java/org/apache/datasketches/GaussianRanks.java
similarity index 66%
rename from src/main/java/org/apache/datasketches/PerformanceUtil.java
rename to src/main/java/org/apache/datasketches/GaussianRanks.java
index cbf872c..2380755 100644
--- a/src/main/java/org/apache/datasketches/PerformanceUtil.java
+++ b/src/main/java/org/apache/datasketches/GaussianRanks.java
@@ -19,13 +19,11 @@
package org.apache.datasketches;
-import static org.apache.datasketches.Util.pwr2LawNext;
-
/**
* @author Lee Rhodes
*/
-public class PerformanceUtil {
- //Quantile fractions computed from the standard normal cumulative distribution.
+public class GaussianRanks {
+ //Fractional ranks computed from the standard normal cumulative distribution.
public static final double M4SD = 0.0000316712418331; //minus 4 StdDev //0
public static final double M3SD = 0.0013498980316301; //minus 3 StdDev //1
public static final double M2SD = 0.0227501319481792; //minus 2 StdDev //2
@@ -39,23 +37,6 @@ public class PerformanceUtil {
{0.0, M4SD, M3SD, M2SD, M1SD, MED, P1SD, P2SD, P3SD, P4SD, 1.0};
public static final int FRACT_LEN = FRACTIONS.length;
- /**
- * Counts the actual number of plotting points between lgStart and lgEnd assuming the given PPO.
- * This is not a simple linear function due to points that may be skipped in the low range.
- * @param lgStart Log2 of the starting value
- * @param lgEnd Log2 of the ending value
- * @param ppo the number of logrithmically evenly spaced points per octave.
- * @return the actual number of plotting points between lgStart and lgEnd.
- */
- public static final int countPoints(final int lgStart, final int lgEnd, final int ppo) {
- int p = 1 << lgStart;
- final int end = 1 << lgEnd;
- int count = 0;
- while (p <= end) {
- p = pwr2LawNext(ppo, p);
- count++;
- }
- return count;
- }
-
+ public static final double[] FRACTIONS_3SD =
+ {0.0, M3SD, M2SD, M1SD, MED, P1SD, P2SD, P3SD, 1.0};
}
diff --git a/src/main/java/org/apache/datasketches/TestingUtil2.java b/src/main/java/org/apache/datasketches/MonotonicPoints.java
similarity index 68%
rename from src/main/java/org/apache/datasketches/TestingUtil2.java
rename to src/main/java/org/apache/datasketches/MonotonicPoints.java
index 504d7d2..599006c 100644
--- a/src/main/java/org/apache/datasketches/TestingUtil2.java
+++ b/src/main/java/org/apache/datasketches/MonotonicPoints.java
@@ -23,22 +23,24 @@ import static java.lang.Math.abs;
import static java.lang.Math.exp;
import static java.lang.Math.log;
import static java.lang.Math.min;
+import static org.apache.datasketches.Util.pwr2LawNext;
import java.util.Random;
-public class TestingUtil2 {
+public class MonotonicPoints {
public static Random rand = new Random();
/**
- *
- * @param x1 given
- * @param x2 given
- * @param points given
- * @param log given
+ * Generates an double[] of monotonic points that are uniformly spaced.
+ * @param x1 one endpoint
+ * @param x2 the other endpoint
+ * @param points to be returned
+ * @param useLog set this true if you want the sequence to be logarithmically spaced. If
+ * true, x1 and x2 must be greater than zero.
* @return double array
*/
public static double[] evenlySpaced(final double x1, final double x2, final int points,
- final boolean log) {
+ final boolean useLog) {
if (points <= 0) {
throw new IllegalArgumentException("points must be > 0");
}
@@ -52,8 +54,8 @@ public class TestingUtil2 {
return out;
}
// 3 or more
- if (log) {
- if ((x2 <= 0) || (x1 <= 0)) {
+ if (useLog) {
+ if (x2 <= 0 || x1 <= 0) {
throw new IllegalArgumentException("x1 and x2 must be > 0.");
}
final double logMin = log(x1);
@@ -71,22 +73,23 @@ public class TestingUtil2 {
}
/**
- *
- * @param x1 given
- * @param x2 given
- * @param points given
- * @param log given
+ * Generates an double[] of monotonic points that are uniform random spaced.
+ * @param x1 one endpoint
+ * @param x2 the other endpoint
+ * @param points to be returned
+ * @param useLog set this true if you want the sequence to be logarithmically spaced. If
+ * true, x1 and x2 must be greater than zero.
* @return double array
*/
public static double[] uniformRandom(final double x1, final double x2, final int points,
- final boolean log) {
+ final boolean useLog) {
if (points <= 0) {
throw new IllegalArgumentException("points must be > 0");
}
final double[] out = new double[points];
- if (log) {
- if ((x2 <= 0) || (x1 <= 0)) {
+ if (useLog) {
+ if (x2 <= 0 || x1 <= 0) {
throw new IllegalArgumentException("x1 and x2 must be > 0.");
}
final double logX1 = log(x1);
@@ -122,7 +125,7 @@ public class TestingUtil2 {
public static final double linXlinYline(final double slope, final double x, final double x0,
final double y0) {
- return mXplusY(slope, (x - x0), y0);
+ return mXplusY(slope, x - x0, y0);
}
public static final double logXlogYline(final double slope, final double x, final double x0,
@@ -131,7 +134,26 @@ public class TestingUtil2 {
}
public static final double mXplusY(final double m, final double x, final double y) {
- return (m * x) + y;
+ return m * x + y;
+ }
+
+ /**
+ * Counts the actual number of plotting points between lgStart and lgEnd assuming the given PPO.
+ * This is not a simple linear function due to points that may be skipped in the low range.
+ * @param lgStart Log2 of the starting value
+ * @param lgEnd Log2 of the ending value
+ * @param ppo the number of logrithmically evenly spaced points per octave.
+ * @return the actual number of plotting points between lgStart and lgEnd.
+ */
+ public static final int countPoints(final int lgStart, final int lgEnd, final int ppo) {
+ int p = 1 << lgStart;
+ final int end = 1 << lgEnd;
+ int count = 0;
+ while (p <= end) {
+ p = pwr2LawNext(ppo, p);
+ count++;
+ }
+ return count;
}
}
diff --git a/src/main/java/org/apache/datasketches/AccuracyStats.java b/src/main/java/org/apache/datasketches/characterization/AccuracyStats.java
similarity index 81%
rename from src/main/java/org/apache/datasketches/AccuracyStats.java
rename to src/main/java/org/apache/datasketches/characterization/AccuracyStats.java
index 87b972e..fd5e2a2 100644
--- a/src/main/java/org/apache/datasketches/AccuracyStats.java
+++ b/src/main/java/org/apache/datasketches/characterization/AccuracyStats.java
@@ -17,25 +17,26 @@
* under the License.
*/
-package org.apache.datasketches;
+package org.apache.datasketches.characterization;
import static org.apache.datasketches.Util.pwr2LawNext;
+import org.apache.datasketches.MonotonicPoints;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
/**
- * Holds key metrics from a single accuracy trial
+ * Holds key metrics from a set of accuracy trials
*
* @author Lee Rhodes
*/
public class AccuracyStats {
- public UpdateDoublesSketch qsk;
+ public UpdateDoublesSketch qsk; //quantile sketch created by constructor
public double sumEst = 0;
public double sumRelErr = 0;
public double sumSqErr = 0;
- public double rmsre = 0;
- public double trueValue;
+ public double rmsre = 0; //used later for plotting
+ public double trueValue; //set by constructor
public int bytes = 0;
/**
@@ -50,19 +51,18 @@ public class AccuracyStats {
/**
* Update
*
- * @param est the value of the estimate for this trial
- * nanoSeconds.
+ * @param est the value of the estimate for a single trial
*/
public void update(final double est) {
qsk.update(est);
sumEst += est;
- sumRelErr += (est / trueValue) - 1.0;
+ sumRelErr += est / trueValue - 1.0;
final double error = est - trueValue;
sumSqErr += error * error;
}
/**
- * Build the Accuracy Stats Array
+ * Build the AccuracyStats Array
* @param lgMin log_base2 of the minimum number of uniques used
* @param lgMax log_base2 of the maximum number of uniques used
* @param ppo the number of points per octave
@@ -71,7 +71,7 @@ public class AccuracyStats {
*/
public static final AccuracyStats[] buildAccuracyStatsArray(
final int lgMin, final int lgMax, final int ppo, final int lgQK) {
- final int qLen = PerformanceUtil.countPoints(lgMin, lgMax, ppo);
+ final int qLen = MonotonicPoints.countPoints(lgMin, lgMax, ppo);
final AccuracyStats[] qArr = new AccuracyStats[qLen];
int p = 1 << lgMin;
for (int i = 0; i < qLen; i++) {
diff --git a/src/main/java/org/apache/datasketches/characterization/BoundsAccuracyStats.java b/src/main/java/org/apache/datasketches/characterization/BoundsAccuracyStats.java
new file mode 100644
index 0000000..9f20b05
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/BoundsAccuracyStats.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization;
+
+import static org.apache.datasketches.Util.pwr2LawNext;
+
+import org.apache.datasketches.MonotonicPoints;
+import org.apache.datasketches.quantiles.DoublesSketchBuilder;
+import org.apache.datasketches.quantiles.UpdateDoublesSketch;
+
+/**
+ * Holds key metrics from a set of accuracy trials
+ * @author Lee Rhodes
+ */
+public class BoundsAccuracyStats {
+ public UpdateDoublesSketch qskEst; //quantile sketch created by constructor
+ public double sumLB3 = 0;
+ public double sumLB2 = 0;
+ public double sumLB1 = 0;
+ public double sumUB1 = 0;
+ public double sumUB2 = 0;
+ public double sumUB3 = 0;
+ public double trueValue; //set by constructor
+
+ public BoundsAccuracyStats(final int k, final int trueValue) {
+ qskEst = new DoublesSketchBuilder().setK(k).build(); //Quantiles
+ this.trueValue = trueValue;
+ }
+
+ /**
+ * Update
+ *
+ * @param est the value of the estimate for a single trial
+ * @param lb3 lower bound1
+ * @param lb2 lower bound2
+ * @param lb1 lower bound3
+ * @param ub1 upper bound1
+ * @param ub2 upper bound2
+ * @param ub3 upper bound3
+ */
+ public void update(
+ final double est,
+ final double lb3,
+ final double lb2,
+ final double lb1,
+ final double ub1,
+ final double ub2,
+ final double ub3) {
+ qskEst.update(est);
+ sumLB3 += lb3;
+ sumLB2 += lb2;
+ sumLB1 += lb1;
+ sumUB1 += ub1;
+ sumUB2 += ub2;
+ sumUB3 += ub3;
+ }
+
+ /**
+ * Build the BoundsAccuracyStats Array
+ * @param lgMin log_base2 of the minimum number of uniques used
+ * @param lgMax log_base2 of the maximum number of uniques used
+ * @param ppo the number of points per octave
+ * @param lgQK the lgK for the Quantiles sketch
+ * @return a BoundsAccuracyStats array
+ */
+ public static final BoundsAccuracyStats[] buildAccuracyStatsArray(
+ final int lgMin, final int lgMax, final int ppo, final int lgQK) {
+ final int qLen = MonotonicPoints.countPoints(lgMin, lgMax, ppo);
+ final BoundsAccuracyStats[] qArr = new BoundsAccuracyStats[qLen];
+ int p = 1 << lgMin;
+ for (int i = 0; i < qLen; i++) {
+ qArr[i] = new BoundsAccuracyStats(1 << lgQK, p);
+ p = pwr2LawNext(ppo, p);
+ }
+ return qArr;
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/fdt/FdtAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/fdt/FdtAccuracyProfile.java
index 46a6fb6..73d4de0 100644
--- a/src/main/java/org/apache/datasketches/characterization/fdt/FdtAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/fdt/FdtAccuracyProfile.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.characterization.fdt;
-import static org.apache.datasketches.PerformanceUtil.FRACTIONS;
+import static org.apache.datasketches.GaussianRanks.FRACTIONS;
import static org.apache.datasketches.PowerLawGenerator.getSlope;
import static org.apache.datasketches.PowerLawGenerator.getY;
import static org.apache.datasketches.Util.pwr2LawNext;
@@ -34,11 +34,11 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.DoublePair;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.Properties;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.fdt.FdtSketch;
import org.apache.datasketches.fdt.Group;
import org.apache.datasketches.fdt.PostProcessor;
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/DruidHllAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/DruidHllAccuracyProfile.java
index 4bbc0dc..ed2259d 100644
--- a/src/main/java/org/apache/datasketches/characterization/hll/DruidHllAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/hll/DruidHllAccuracyProfile.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.characterization.hll;
-import org.apache.datasketches.AccuracyStats;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.druid.hll.HyperLogLogCollector;
import org.apache.druid.hll.HyperLogLogHash;
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/HllAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/HllAccuracyProfile.java
index 5612720..c51116e 100644
--- a/src/main/java/org/apache/datasketches/characterization/hll/HllAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/hll/HllAccuracyProfile.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.characterization.hll;
-import org.apache.datasketches.AccuracyStats;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/HllBoundsAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/HllBoundsAccuracyProfile.java
new file mode 100644
index 0000000..d0a4233
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/HllBoundsAccuracyProfile.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.characterization.BoundsAccuracyStats;
+import org.apache.datasketches.characterization.uniquecount.BaseBoundsAccuracyProfile;
+import org.apache.datasketches.hll.HllSketch;
+import org.apache.datasketches.hll.TgtHllType;
+import org.apache.datasketches.memory.WritableMemory;
+
+/**
+ * @author Lee Rhodes
+ */
+public class HllBoundsAccuracyProfile extends BaseBoundsAccuracyProfile {
+ private HllSketch sketch;
+ private boolean useComposite; //accuracy, HLL
+
+ @Override
+ public void configure() {
+ //Configure Sketch
+ final boolean direct = Boolean.parseBoolean(prop.mustGet("HLL_direct"));
+ useComposite = Boolean.parseBoolean(prop.mustGet("HLL_useComposite"));
+
+ final TgtHllType tgtHllType;
+ final String type = prop.mustGet("HLL_tgtHllType");
+ if (type.equalsIgnoreCase("HLL4")) { tgtHllType = TgtHllType.HLL_4; }
+ else if (type.equalsIgnoreCase("HLL6")) { tgtHllType = TgtHllType.HLL_6; }
+ else { tgtHllType = TgtHllType.HLL_8; }
+
+ if (direct) {
+ final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+ final WritableMemory wmem = WritableMemory.allocate(bytes);
+ sketch = new HllSketch(lgK, tgtHllType, wmem);
+ } else {
+ sketch = new HllSketch(lgK, tgtHllType);
+ }
+ }
+
+ @Override
+ public void doTrial() {
+ final int qArrLen = qArr.length;
+ sketch.reset(); //reuse the same sketch
+ long lastUniques = 0;
+ if (useComposite) {
+ for (int i = 0; i < qArrLen; i++) {
+ final BoundsAccuracyStats q = qArr[i];
+ final long delta = (long)(q.trueValue - lastUniques);
+ for (long u = 0; u < delta; u++) {
+ sketch.update(++vIn);
+ }
+ lastUniques += delta;
+ final double est = sketch.getCompositeEstimate();
+ q.update(est,
+ sketch.getLowerBound(3), sketch.getLowerBound(2), sketch.getLowerBound(1),
+ sketch.getUpperBound(1), sketch.getUpperBound(2), sketch.getUpperBound(3));
+ }
+ } else { //use longs; use HIP
+ for (int i = 0; i < qArrLen; i++) {
+ final BoundsAccuracyStats q = qArr[i];
+ final long delta = (long)(q.trueValue - lastUniques);
+ for (long u = 0; u < delta; u++) {
+ sketch.update(++vIn);
+ }
+ lastUniques += delta;
+ final double est = sketch.getEstimate();
+ q.update(est, sketch.getLowerBound(3), sketch.getLowerBound(2), sketch.getLowerBound(1),
+ sketch.getUpperBound(1), sketch.getUpperBound(2), sketch.getUpperBound(3));
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java
index 804af50..2d8efa4 100644
--- a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.characterization.hll;
-import org.apache.datasketches.AccuracyStats;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import com.google.zetasketch.HyperLogLogPlusPlus;
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/BaseQuantilesAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/BaseQuantilesAccuracyProfile.java
index 233e822..a122afa 100644
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/BaseQuantilesAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/BaseQuantilesAccuracyProfile.java
@@ -23,7 +23,7 @@ import static org.apache.datasketches.Util.pwr2LawNext;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
-import org.apache.datasketches.PerformanceUtil;
+import org.apache.datasketches.MonotonicPoints;
import org.apache.datasketches.Properties;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
@@ -68,7 +68,7 @@ public abstract class BaseQuantilesAccuracyProfile implements JobProfile {
job.println("StreamLength\tError");
- final int numSteps = PerformanceUtil.countPoints(lgMin, lgMax, ppo);
+ final int numSteps = MonotonicPoints.countPoints(lgMin, lgMax, ppo);
int streamLength = 1 << lgMin;
for (int i = 0; i < numSteps; i++) {
prepareTrial(streamLength);
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/tdigest/QuantilesAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/tdigest/QuantilesAccuracyProfile.java
index 579969e..edc0eb9 100644
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/tdigest/QuantilesAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/tdigest/QuantilesAccuracyProfile.java
@@ -23,7 +23,7 @@ import static org.apache.datasketches.Util.pwr2LawNext;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
-import org.apache.datasketches.PerformanceUtil;
+import org.apache.datasketches.MonotonicPoints;
import org.apache.datasketches.Properties;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
@@ -60,7 +60,7 @@ public abstract class QuantilesAccuracyProfile implements JobProfile {
job.println("StreamLength\tError");
- final int numSteps = PerformanceUtil.countPoints(lgMin, lgMax, ppo);
+ final int numSteps = MonotonicPoints.countPoints(lgMin, lgMax, ppo);
int streamLength = 1 << lgMin;
for (int i = 0; i < numSteps; i++) {
prepareTrial(streamLength);
diff --git a/src/main/java/org/apache/datasketches/characterization/theta/ThetaAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/theta/ThetaAccuracyProfile.java
index bf65a83..0b81f82 100644
--- a/src/main/java/org/apache/datasketches/characterization/theta/ThetaAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/theta/ThetaAccuracyProfile.java
@@ -19,9 +19,9 @@
package org.apache.datasketches.characterization.theta;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.Family;
import org.apache.datasketches.ResizeFactor;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.theta.Sketch;
diff --git a/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile.java
index f2525b6..353bd2b 100644
--- a/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.characterization.theta;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.ResizeFactor;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.theta.SetOperationBuilder;
diff --git a/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile2.java b/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile2.java
index 58ccb2a..ce6cbcd 100644
--- a/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile2.java
+++ b/src/main/java/org/apache/datasketches/characterization/theta/ThetaUnionAccuracyProfile2.java
@@ -19,9 +19,9 @@
package org.apache.datasketches.characterization.theta;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.Family;
import org.apache.datasketches.ResizeFactor;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.theta.SetOperationBuilder;
diff --git a/src/main/java/org/apache/datasketches/characterization/theta/concurrent/ConcurrentThetaAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/theta/concurrent/ConcurrentThetaAccuracyProfile.java
index 4cad487..2c800e2 100644
--- a/src/main/java/org/apache/datasketches/characterization/theta/concurrent/ConcurrentThetaAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/theta/concurrent/ConcurrentThetaAccuracyProfile.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.characterization.theta.concurrent;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
-import org.apache.datasketches.AccuracyStats;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
import org.apache.datasketches.memory.WritableDirectHandle;
import org.apache.datasketches.memory.WritableMemory;
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java
index 713c473..8773be8 100644
--- a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java
@@ -19,17 +19,17 @@
package org.apache.datasketches.characterization.uniquecount;
-import static org.apache.datasketches.PerformanceUtil.FRACTIONS;
-import static org.apache.datasketches.PerformanceUtil.FRACT_LEN;
+import static org.apache.datasketches.GaussianRanks.FRACTIONS;
+import static org.apache.datasketches.GaussianRanks.FRACT_LEN;
import static org.apache.datasketches.Util.milliSecToString;
import static org.apache.datasketches.Util.pwr2LawNext;
import java.io.PrintWriter;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.Properties;
+import org.apache.datasketches.characterization.AccuracyStats;
import org.apache.datasketches.quantiles.DoublesSketch;
/**
@@ -71,7 +71,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
qArr = AccuracyStats.buildAccuracyStatsArray(lgMinU, lgMaxU, uPPO, lgQK);
lgK = Integer.parseInt(prop.mustGet("LgK"));
final String getSizeStr = prop.get("Trials_bytes");
- getSize = (getSizeStr == null) ? false : Boolean.parseBoolean(getSizeStr);
+ getSize = getSizeStr == null ? false : Boolean.parseBoolean(getSizeStr);
configure();
doTrials();
shutdown();
@@ -122,7 +122,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
//This will generate a table of data up for each intermediate Trials point
int lastT = 0;
while (lastT < maxT) {
- final int nextT = (lastT == 0) ? minT : pwr2LawNext(tPPO, lastT);
+ final int nextT = lastT == 0 ? minT : pwr2LawNext(tPPO, lastT);
final int delta = nextT - lastT;
for (int i = 0; i < delta; i++) {
doTrial();
@@ -147,8 +147,8 @@ public abstract class BaseAccuracyProfile implements JobProfile {
final long currentTime_mS = System.currentTimeMillis();
final long cumTime_mS = currentTime_mS - job.getStartTime();
println("Cum Time : " + milliSecToString(cumTime_mS));
- final double timePerTrial_mS = (cumTime_mS * 1.0) / lastT;
- final double avgUpdateTime_ns = (timePerTrial_mS * 1e6) / maxU;
+ final double timePerTrial_mS = cumTime_mS * 1.0 / lastT;
+ final double avgUpdateTime_ns = timePerTrial_mS * 1e6 / maxU;
println("Time Per Trial, mSec : " + timePerTrial_mS);
println("Avg Update Time, nSec : " + avgUpdateTime_ns);
println("Date Time : "
@@ -199,7 +199,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
//Quantiles
final double[] quants = qArr[pt].qsk.getQuantiles(FRACTIONS);
for (int i = 0; i < FRACT_LEN; i++) {
- sb.append((quants[i] / uniques) - 1.0).append(TAB);
+ sb.append(quants[i] / uniques - 1.0).append(TAB);
}
if (getSize) {
sb.append(bytes).append(TAB);
@@ -267,7 +267,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
}
/**
- *
+ * This removes possible duplicate values in the given splitpoints array.
* @param splitPoints the given splitPoints
* @return the reduced array of splitPoints
*/
@@ -276,7 +276,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
double lastV = splitPoints[0];
for (int i = 0; i < splitPoints.length; i++) {
final double v = splitPoints[i];
- if (v <= lastV) { continue; }
+ if (v <= lastV) { continue; } //duplicate
num++;
lastV = v;
}
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
similarity index 75%
copy from src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java
copy to src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
index 713c473..29dde4e 100644
--- a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
@@ -19,23 +19,22 @@
package org.apache.datasketches.characterization.uniquecount;
-import static org.apache.datasketches.PerformanceUtil.FRACTIONS;
-import static org.apache.datasketches.PerformanceUtil.FRACT_LEN;
+import static org.apache.datasketches.GaussianRanks.FRACTIONS_3SD;
import static org.apache.datasketches.Util.milliSecToString;
import static org.apache.datasketches.Util.pwr2LawNext;
import java.io.PrintWriter;
-import org.apache.datasketches.AccuracyStats;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.Properties;
+import org.apache.datasketches.characterization.BoundsAccuracyStats;
import org.apache.datasketches.quantiles.DoublesSketch;
/**
* @author Lee Rhodes
*/
-public abstract class BaseAccuracyProfile implements JobProfile {
+public abstract class BaseBoundsAccuracyProfile implements JobProfile {
Job job;
PrintWriter pw;
public Properties prop;
@@ -51,7 +50,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
boolean interData;
boolean postPMFs;
public boolean getSize = false;
- public AccuracyStats[] qArr;
+ public BoundsAccuracyStats[] qArr;
//JobProfile
@Override
@@ -68,10 +67,8 @@ public abstract class BaseAccuracyProfile implements JobProfile {
postPMFs = Boolean.parseBoolean(prop.mustGet("Trials_postPMFs"));
uPPO = Integer.parseInt(prop.mustGet("Trials_UPPO"));
lgQK = Integer.parseInt(prop.mustGet("Trials_lgQK"));
- qArr = AccuracyStats.buildAccuracyStatsArray(lgMinU, lgMaxU, uPPO, lgQK);
+ qArr = BoundsAccuracyStats.buildAccuracyStatsArray(lgMinU, lgMaxU, uPPO, lgQK);
lgK = Integer.parseInt(prop.mustGet("LgK"));
- final String getSizeStr = prop.get("Trials_bytes");
- getSize = (getSizeStr == null) ? false : Boolean.parseBoolean(getSizeStr);
configure();
doTrials();
shutdown();
@@ -122,7 +119,7 @@ public abstract class BaseAccuracyProfile implements JobProfile {
//This will generate a table of data up for each intermediate Trials point
int lastT = 0;
while (lastT < maxT) {
- final int nextT = (lastT == 0) ? minT : pwr2LawNext(tPPO, lastT);
+ final int nextT = lastT == 0 ? minT : pwr2LawNext(tPPO, lastT);
final int delta = nextT - lastT;
for (int i = 0; i < delta; i++) {
doTrial();
@@ -132,12 +129,12 @@ public abstract class BaseAccuracyProfile implements JobProfile {
if (nextT < maxT) { // intermediate
if (interData) {
job.println(getHeader());
- process(getSize, qArr, lastT, sb);
+ process(qArr, lastT, sb);
job.println(sb.toString());
}
} else { //done
job.println(getHeader());
- process(getSize, qArr, lastT, sb);
+ process(qArr, lastT, sb);
job.println(sb.toString());
}
@@ -147,8 +144,8 @@ public abstract class BaseAccuracyProfile implements JobProfile {
final long currentTime_mS = System.currentTimeMillis();
final long cumTime_mS = currentTime_mS - job.getStartTime();
println("Cum Time : " + milliSecToString(cumTime_mS));
- final double timePerTrial_mS = (cumTime_mS * 1.0) / lastT;
- final double avgUpdateTime_ns = (timePerTrial_mS * 1e6) / maxU;
+ final double timePerTrial_mS = cumTime_mS * 1.0 / lastT;
+ final double avgUpdateTime_ns = timePerTrial_mS * 1e6 / maxU;
println("Time Per Trial, mSec : " + timePerTrial_mS);
println("Avg Update Time, nSec : " + avgUpdateTime_ns);
println("Date Time : "
@@ -168,64 +165,43 @@ public abstract class BaseAccuracyProfile implements JobProfile {
}
}
- private static void process(final boolean getSize, final AccuracyStats[] qArr,
+ private static void process(final BoundsAccuracyStats[] qArr,
final int cumTrials, final StringBuilder sb) {
final int points = qArr.length;
sb.setLength(0);
for (int pt = 0; pt < points; pt++) {
- final AccuracyStats q = qArr[pt];
- final double uniques = q.trueValue;
- final double meanEst = q.sumEst / cumTrials;
- final double meanRelErr = q.sumRelErr / cumTrials;
- final double meanSqErr = q.sumSqErr / cumTrials; //intermediate
- final double normMeanSqErr = meanSqErr / (1.0 * uniques * uniques); //intermediate
- final double rmsRelErr = Math.sqrt(normMeanSqErr);
- q.rmsre = rmsRelErr;
- final int bytes = q.bytes;
+ final BoundsAccuracyStats q = qArr[pt];
//OUTPUT
- //sb.setLength(0);
- sb.append(uniques).append(TAB);
-
- //Sketch meanEst, meanEstErr, norm RMS Err
- sb.append(meanEst).append(TAB);
- sb.append(meanRelErr).append(TAB);
- sb.append(rmsRelErr).append(TAB);
-
+ sb.append(q.trueValue).append(TAB);
//TRIALS
sb.append(cumTrials).append(TAB);
//Quantiles
- final double[] quants = qArr[pt].qsk.getQuantiles(FRACTIONS);
- for (int i = 0; i < FRACT_LEN; i++) {
- sb.append((quants[i] / uniques) - 1.0).append(TAB);
- }
- if (getSize) {
- sb.append(bytes).append(TAB);
- sb.append(rmsRelErr * Math.sqrt(bytes));
- } else {
- sb.append(0).append(TAB);
- sb.append(0);
+ final double[] quants = q.qskEst.getQuantiles(FRACTIONS_3SD);
+ final int len = quants.length;
+ for (int i = 0; i < len; i++) {
+ sb.append(quants[i] / q.trueValue - 1.0).append(TAB);
}
+ //Bound averages
+ sb.append(q.sumLB3 / cumTrials).append(TAB);
+ sb.append(q.sumLB2 / cumTrials).append(TAB);
+ sb.append(q.sumLB1 / cumTrials).append(TAB);
+ sb.append(q.sumUB1 / cumTrials).append(TAB);
+ sb.append(q.sumUB2 / cumTrials).append(TAB);
+ sb.append(q.sumUB3 / cumTrials).append(TAB);
sb.append(LS);
}
}
private static String getHeader() {
final StringBuilder sb = new StringBuilder();
- sb.append("InU").append(TAB); //col 1
- //Estimates
- sb.append("MeanEst").append(TAB); //col 2
- sb.append("MeanRelErr").append(TAB); //col 3
- sb.append("RMS_RE").append(TAB); //col 4
-
+ sb.append("InU").append(TAB);
//Trials
- sb.append("Trials").append(TAB); //col 5
-
+ sb.append("Trials").append(TAB);
//Quantiles
sb.append("Min").append(TAB);
- sb.append("Q(.0000317)").append(TAB);
sb.append("Q(.00135)").append(TAB);
sb.append("Q(.02275)").append(TAB);
sb.append("Q(.15866)").append(TAB);
@@ -233,10 +209,13 @@ public abstract class BaseAccuracyProfile implements JobProfile {
sb.append("Q(.84134)").append(TAB);
sb.append("Q(.97725)").append(TAB);
sb.append("Q(.99865)").append(TAB);
- sb.append("Q(.9999683)").append(TAB);
sb.append("Max").append(TAB);
- sb.append("Bytes").append(TAB);
- sb.append("ReMerit");
+ sb.append("avgLB3").append(TAB);
+ sb.append("avgLB2").append(TAB);
+ sb.append("avgLB1").append(TAB);
+ sb.append("avgUB1").append(TAB);
+ sb.append("avgUB2").append(TAB);
+ sb.append("avgUB3").append(TAB);
return sb.toString();
}
@@ -244,9 +223,9 @@ public abstract class BaseAccuracyProfile implements JobProfile {
* Outputs the Probability Mass Function given the AccuracyStats.
* @param q the given AccuracyStats
*/
- private static String outputPMF(final AccuracyStats q) {
- final DoublesSketch qSk = q.qsk;
- final double[] splitPoints = qSk.getQuantiles(FRACTIONS); //1:1
+ private static String outputPMF(final BoundsAccuracyStats q) {
+ final DoublesSketch qSk = q.qskEst;
+ final double[] splitPoints = qSk.getQuantiles(FRACTIONS_3SD); //1:1
final double[] reducedSp = reduceSplitPoints(splitPoints);
final double[] pmfArr = qSk.getPMF(reducedSp); //pmfArr is one larger
final long trials = qSk.getN();
diff --git a/src/main/java/org/apache/datasketches/hll/HllRedirects.java b/src/main/java/org/apache/datasketches/hll/HllRedirects.java
new file mode 100644
index 0000000..7261353
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll/HllRedirects.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll;
+
+import static org.apache.datasketches.hll.HllEstimators.hllCompositeEstimate;
+import static org.apache.datasketches.hll.HllEstimators.hllLowerBound;
+import static org.apache.datasketches.hll.HllEstimators.hllUpperBound;
+import static org.apache.datasketches.hll.RelativeErrorTables.getRelErr;
+
+/**
+ * Provide access to hidden HLL methods.
+ *
+ * @author Lee Rhodes
+ */
+public class HllRedirects {
+
+ public double getRelativeError(final boolean upperBound, final boolean oooFlag,
+ final int lgK, final int stdDev) {
+ return getRelErr(upperBound, oooFlag, lgK, stdDev);
+ }
+
+ public double getHllLowerBound(final HllSketch sk, final int numStdDev) {
+ return hllLowerBound((AbstractHllArray)sk.hllSketchImpl, numStdDev);
+ }
+
+ public double getHllUpperBound(final HllSketch sk, final int numStdDev) {
+ return hllUpperBound((AbstractHllArray)sk.hllSketchImpl, numStdDev);
+ }
+
+ public double getHIPEstimate(final HllSketch sk) {
+ return getAbstractHllArray(sk).getHipEstimate();
+ }
+
+ public double getCompositeEstimate(final HllSketch sk) {
+ return hllCompositeEstimate((AbstractHllArray)sk.hllSketchImpl);
+ }
+
+ public AbstractHllArray getAbstractHllArray(final HllSketch sk) {
+ return (AbstractHllArray)sk.hllSketchImpl;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/hll/package-info.java b/src/main/java/org/apache/datasketches/hll/package-info.java
new file mode 100644
index 0000000..62238b8
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/hll/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @author Lee Rhodes
+ */
+
+package org.apache.datasketches.hll;
diff --git a/src/main/resources/hll/HllBoundsAccuracyJob.conf b/src/main/resources/hll/HllBoundsAccuracyJob.conf
new file mode 100644
index 0000000..7169b85
--- /dev/null
+++ b/src/main/resources/hll/HllBoundsAccuracyJob.conf
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+
+#Uniques Profile
+Trials_lgMinU=0 #The starting # of uniques that is printed at the end.
+Trials_lgMaxU=12 #How high the # uniques go
+Trials_UPPO=16 #The horizontal x-resolution of trials points
+
+
+# Trials Profile
+Trials_lgMinT=8 #prints intermediate results starting w/ this lgMinT
+Trials_lgMaxT=10 #The max trials
+Trials_TPPO=1 #how often intermediate results are printed
+
+Trials_lgQK=12 #size of quantiles sketch
+Trials_interData=true
+Trials_postPMFs=false
+
+# Date-Time Profile
+TimeZone=PST
+TimeZoneOffset=-28800000 # offset in millisec
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
+
+#Job Profile
+JobProfile=org.apache.datasketches.characterization.hll.HllBoundsAccuracyProfile
+LgK=12
+HLL_direct=false #only for Theta, HLL. See javadocs.
+HLL_tgtHllType=HLL8
+HLL_useComposite=false
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[incubator-datasketches-characterization] 02/02: Update Bounds tests
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch HllBoundsTesting
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-characterization.git
commit d3c9454bf2c7731f29b2c1027eaaaaa2607969d9
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Wed Apr 1 10:34:10 2020 -0700
Update Bounds tests
---
.../uniquecount/BaseBoundsAccuracyProfile.java | 23 ++++++++++++++--------
src/main/resources/hll/HllBoundsAccuracyJob.conf | 8 ++++----
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
index 29dde4e..583a0c5 100644
--- a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseBoundsAccuracyProfile.java
@@ -172,9 +172,16 @@ public abstract class BaseBoundsAccuracyProfile implements JobProfile {
sb.setLength(0);
for (int pt = 0; pt < points; pt++) {
final BoundsAccuracyStats q = qArr[pt];
+ final double uniques = q.trueValue;
+ final double relLb3 = q.sumLB3 / cumTrials / uniques - 1.0;
+ final double relLb2 = q.sumLB2 / cumTrials / uniques - 1.0;
+ final double relLb1 = q.sumLB1 / cumTrials / uniques - 1.0;
+ final double relUb1 = q.sumUB1 / cumTrials / uniques - 1.0;
+ final double relUb2 = q.sumUB2 / cumTrials / uniques - 1.0;
+ final double relUb3 = q.sumUB3 / cumTrials / uniques - 1.0;
//OUTPUT
- sb.append(q.trueValue).append(TAB);
+ sb.append(uniques).append(TAB);
//TRIALS
sb.append(cumTrials).append(TAB);
@@ -182,15 +189,15 @@ public abstract class BaseBoundsAccuracyProfile implements JobProfile {
final double[] quants = q.qskEst.getQuantiles(FRACTIONS_3SD);
final int len = quants.length;
for (int i = 0; i < len; i++) {
- sb.append(quants[i] / q.trueValue - 1.0).append(TAB);
+ sb.append(quants[i] / uniques - 1.0).append(TAB);
}
//Bound averages
- sb.append(q.sumLB3 / cumTrials).append(TAB);
- sb.append(q.sumLB2 / cumTrials).append(TAB);
- sb.append(q.sumLB1 / cumTrials).append(TAB);
- sb.append(q.sumUB1 / cumTrials).append(TAB);
- sb.append(q.sumUB2 / cumTrials).append(TAB);
- sb.append(q.sumUB3 / cumTrials).append(TAB);
+ sb.append(relLb3).append(TAB);
+ sb.append(relLb2).append(TAB);
+ sb.append(relLb1).append(TAB);
+ sb.append(relUb1).append(TAB);
+ sb.append(relUb2).append(TAB);
+ sb.append(relUb3).append(TAB);
sb.append(LS);
}
}
diff --git a/src/main/resources/hll/HllBoundsAccuracyJob.conf b/src/main/resources/hll/HllBoundsAccuracyJob.conf
index 7169b85..6d6455d 100644
--- a/src/main/resources/hll/HllBoundsAccuracyJob.conf
+++ b/src/main/resources/hll/HllBoundsAccuracyJob.conf
@@ -19,13 +19,13 @@
#Uniques Profile
Trials_lgMinU=0 #The starting # of uniques that is printed at the end.
-Trials_lgMaxU=12 #How high the # uniques go
+Trials_lgMaxU=20 #How high the # uniques go
Trials_UPPO=16 #The horizontal x-resolution of trials points
# Trials Profile
-Trials_lgMinT=8 #prints intermediate results starting w/ this lgMinT
-Trials_lgMaxT=10 #The max trials
+Trials_lgMinT=10 #prints intermediate results starting w/ this lgMinT
+Trials_lgMaxT=20 #The max trials
Trials_TPPO=1 #how often intermediate results are printed
Trials_lgQK=12 #size of quantiles sketch
@@ -40,7 +40,7 @@ ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
#Job Profile
JobProfile=org.apache.datasketches.characterization.hll.HllBoundsAccuracyProfile
-LgK=12
+LgK=4
HLL_direct=false #only for Theta, HLL. See javadocs.
HLL_tgtHllType=HLL8
HLL_useComposite=false
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org