You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/04/18 00:50:40 UTC
[datasketches-characterization] branch master updated: update characterization files
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/datasketches-characterization.git
The following commit(s) were added to refs/heads/master by this push:
new c23b879 update characterization files
c23b879 is described below
commit c23b879158dfe85267b49dce0bc518beaad783f0
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Sun Apr 17 17:50:33 2022 -0700
update characterization files
---
...llDoublesSketchRankGaussianAccuracyProfile.java | 5 +-
...KllFloatsSketchRankGaussianAccuracyProfile.java | 6 +-
.../quantiles/KllSketchAccuracyProfile.java | 109 ++++++++++++---------
.../resources/quantiles/KllSketchAccuracyJob.conf | 1 +
4 files changed, 63 insertions(+), 58 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/KllDoublesSketchRankGaussianAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/KllDoublesSketchRankGaussianAccuracyProfile.java
index 3969d44..3c53e94 100644
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/KllDoublesSketchRankGaussianAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/KllDoublesSketchRankGaussianAccuracyProfile.java
@@ -184,10 +184,7 @@ public class KllDoublesSketchRankGaussianAccuracyProfile implements JobProfile {
//Do numTrials for all plot points
for (int t = 0; t < numTrials; t++) {
- final WritableMemory wmem = WritableMemory.allocate(10000);
- sk = KllDoublesSketch.newDirectInstance(k, wmem, memReqSvr);
- // sk = KllDoublesSketch.newHeapInstance(k);
- // sk = new KllDoublesSketch(K);
+ sk.reset();
doTrial(sk, stream, trueValues, corrTrueValues, errQSkArr);
}
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchRankGaussianAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchRankGaussianAccuracyProfile.java
index 9c9e7bd..89454f9 100644
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchRankGaussianAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchRankGaussianAccuracyProfile.java
@@ -28,7 +28,6 @@ import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.MonotonicPoints;
import org.apache.datasketches.characterization.Shuffle;
-import org.apache.datasketches.kll.KllDoublesSketch;
import org.apache.datasketches.kll.KllFloatsSketch;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
@@ -186,10 +185,7 @@ public class KllFloatsSketchRankGaussianAccuracyProfile implements JobProfile {
//Do numTrials for all plot points
for (int t = 0; t < numTrials; t++) {
- final WritableMemory wmem = WritableMemory.allocate(10000);
- sk = KllFloatsSketch.newDirectInstance(k, wmem, memReqSvr);
- //sk = KllFloatsSketch.newHeapInstance(k);
- //sk = new KllFloatsSketch(K);
+ sk.reset();
doTrial(sk, stream, trueValues, corrTrueValues, errQSkArr);
}
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
index 54429f6..6e2061f 100644
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
@@ -41,47 +41,67 @@ public class KllSketchAccuracyProfile extends BaseQuantilesAccuracyProfile {
private double[] doubleQueryValues;
private boolean useBulk;
private String type;
- private boolean useFloat;
- private boolean useDouble;
+ private boolean direct = false;
+ private boolean useDouble = false; //useFloat is default
//add other types
+ KllDoublesSketch dsk = null;
+ KllFloatsSketch fsk = null;
+
@Override
void configure(final Properties props) {
- useFloat = useDouble = false;
k = Integer.parseInt(props.mustGet("K"));
useBulk = Boolean.parseBoolean(props.mustGet("useBulk"));
+ direct = Boolean.parseBoolean(props.mustGet("direct"));
type = props.mustGet("type");
- if (type.equalsIgnoreCase("float")) { useFloat = true; }
- else if (type.equalsIgnoreCase("double")) { useDouble = true; }
- else { throw new IllegalArgumentException("Unknown primitive type: " + type); }
+ if (type.equalsIgnoreCase("double")) { useDouble = true; }
+
+ configureSketch();
+ }
+
+ void configureSketch() {
+ if (useDouble) {
+ if (direct) {
+ final WritableMemory dstMem = WritableMemory.allocate(10000);
+ dsk = KllDoublesSketch.newDirectInstance(k, dstMem, memReqSvr);
+ } else { //heap
+ dsk = KllDoublesSketch.newHeapInstance(k);
+ }
+ } else { //useFloat
+ if (direct) {
+ final WritableMemory dstMem = WritableMemory.allocate(10000);
+ fsk = KllFloatsSketch.newDirectInstance(k, dstMem, memReqSvr);
+ } else { //heap
+ fsk = KllFloatsSketch.newHeapInstance(k);
+ }
+ }
}
@Override
void prepareTrial(final int streamLength) {
// prepare input data that will be permuted
- if (useFloat) {
- inputFloatValues = new float[streamLength];
+ if (useDouble) {
+ inputDoubleValues = new double[streamLength];
for (int i = 0; i < streamLength; i++) {
- inputFloatValues[i] = i;
+ inputDoubleValues[i] = i;
}
if (useBulk) {
// prepare query data that must remain ordered
- floatQueryValues = new float[streamLength];
+ doubleQueryValues = new double[streamLength];
for (int i = 0; i < streamLength; i++) {
- floatQueryValues[i] = i;
+ doubleQueryValues[i] = i;
}
}
- }
- else if (useDouble) {
- inputDoubleValues = new double[streamLength];
+ } else { //use Float
+ inputFloatValues = new float[streamLength];
for (int i = 0; i < streamLength; i++) {
- inputDoubleValues[i] = i;
+ inputFloatValues[i] = i;
}
if (useBulk) {
// prepare query data that must remain ordered
- doubleQueryValues = new double[streamLength];
+ floatQueryValues = new float[streamLength];
for (int i = 0; i < streamLength; i++) {
- doubleQueryValues[i] = i;
+ floatQueryValues[i] = i;
}
}
}
@@ -90,58 +110,49 @@ public class KllSketchAccuracyProfile extends BaseQuantilesAccuracyProfile {
@Override
double doTrial() {
double maxRankError = 0;
- if (useFloat) {
- shuffle(inputFloatValues);
- // build sketch
- final WritableMemory wmem = WritableMemory.allocate(10000);
- final KllFloatsSketch sketch;
- sketch = KllFloatsSketch.newDirectInstance(k, wmem, memReqSvr);
- //sketch = KllFloatsSketch.newHeapInstance(k);
- //sketch = new KllFloatsSketch(k);
- for (int i = 0; i < inputFloatValues.length; i++) {
- sketch.update(inputFloatValues[i]);
+ if (useDouble) {
+ shuffle(inputDoubleValues);
+ // reset sketch
+ dsk.reset();
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ dsk.update(inputDoubleValues[i]);
}
// query sketch and gather results
maxRankError = 0;
if (useBulk) {
- final double[] estRanks = sketch.getCDF(floatQueryValues);
- for (int i = 0; i < inputFloatValues.length; i++) {
- final double trueRank = (double) i / inputFloatValues.length;
+ final double[] estRanks = dsk.getCDF(doubleQueryValues);
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ final double trueRank = (double) i / inputDoubleValues.length;
maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRanks[i]));
}
} else {
- for (int i = 0; i < inputFloatValues.length; i++) {
- final double trueRank = (double) i / inputFloatValues.length;
- final double estRank = sketch.getRank(i);
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ final double trueRank = (double) i / inputDoubleValues.length;
+ final double estRank = dsk.getRank(i);
maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRank));
}
}
- }
- else if (useDouble) {
- shuffle(inputDoubleValues);
+ } else { //use Float
+ shuffle(inputFloatValues);
// build sketch
- final WritableMemory wmem = WritableMemory.allocate(10000);
- final KllDoublesSketch sketch;
- sketch = KllDoublesSketch.newDirectInstance(k, wmem, memReqSvr);
- //sketch = KllDoublesSketch.newHeapInstance(k);
- //sketch = new KllDoublesSketch(k);
- for (int i = 0; i < inputDoubleValues.length; i++) {
- sketch.update(inputDoubleValues[i]);
+ fsk.reset();
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ fsk.update(inputFloatValues[i]);
}
// query sketch and gather results
maxRankError = 0;
if (useBulk) {
- final double[] estRanks = sketch.getCDF(doubleQueryValues);
- for (int i = 0; i < inputDoubleValues.length; i++) {
- final double trueRank = (double) i / inputDoubleValues.length;
+ final double[] estRanks = fsk.getCDF(floatQueryValues);
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ final double trueRank = (double) i / inputFloatValues.length;
maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRanks[i]));
}
} else {
- for (int i = 0; i < inputDoubleValues.length; i++) {
- final double trueRank = (double) i / inputDoubleValues.length;
- final double estRank = sketch.getRank(i);
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ final double trueRank = (double) i / inputFloatValues.length;
+ final double estRank = fsk.getRank(i);
maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRank));
}
}
diff --git a/src/main/resources/quantiles/KllSketchAccuracyJob.conf b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
index 63107bd..0d7b563 100644
--- a/src/main/resources/quantiles/KllSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
@@ -25,3 +25,4 @@ lgMin=0 # The starting stream length
lgMax=23 # How high the stream length goes
PPO=8 # The horizontal x-resolution of trials points
type=double
+direct=true
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org