You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/02/08 01:56:37 UTC
[datasketches-characterization] 03/05: update KLL characterizations
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/datasketches-characterization.git
commit 554edf09c1c8771e01fb6d75a05a4626ac81bcc7
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Feb 7 17:50:54 2022 -0800
update KLL characterizations
---
.../quantiles/KllFloatsSketchAccuracyProfile.java | 83 ------------
.../quantiles/KllSketchAccuracyProfile.java | 142 +++++++++++++++++++++
...hAccuracyJob.conf => KllSketchAccuracyJob.conf} | 3 +-
3 files changed, 144 insertions(+), 84 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchAccuracyProfile.java
deleted file mode 100644
index 56cd7dd..0000000
--- a/src/main/java/org/apache/datasketches/characterization/quantiles/KllFloatsSketchAccuracyProfile.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.characterization.quantiles;
-
-import static org.apache.datasketches.characterization.Shuffle.shuffle;
-
-import org.apache.datasketches.Properties;
-import org.apache.datasketches.kll.KllFloatsSketch;
-
-public class KllFloatsSketchAccuracyProfile extends BaseQuantilesAccuracyProfile {
-
- private int k;
- private float[] inputValues;
- private float[] queryValues;
- private boolean useBulk;
-
- @Override
- void configure(final Properties props) {
- k = Integer.parseInt(props.mustGet("K"));
- useBulk = Boolean.parseBoolean(props.mustGet("useBulk"));
- }
-
- @Override
- void prepareTrial(final int streamLength) {
- // prepare input data that will be permuted
- inputValues = new float[streamLength];
- for (int i = 0; i < streamLength; i++) {
- inputValues[i] = i;
- }
- if (useBulk) {
- // prepare query data that must remain ordered
- queryValues = new float[streamLength];
- for (int i = 0; i < streamLength; i++) {
- queryValues[i] = i;
- }
- }
- }
-
- @Override
- double doTrial() {
- shuffle(inputValues);
-
- // build sketch
- final KllFloatsSketch sketch = new KllFloatsSketch(k);
- for (int i = 0; i < inputValues.length; i++) {
- sketch.update(inputValues[i]);
- }
-
- // query sketch and gather results
- double maxRankError = 0;
- if (useBulk) {
- final double[] estRanks = sketch.getCDF(queryValues);
- for (int i = 0; i < inputValues.length; i++) {
- final double trueRank = (double) i / inputValues.length;
- maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRanks[i]));
- }
- } else {
- for (int i = 0; i < inputValues.length; i++) {
- final double trueRank = (double) i / inputValues.length;
- final double estRank = sketch.getRank(i);
- maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRank));
- }
- }
- return maxRankError;
- }
-}
diff --git a/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
new file mode 100644
index 0000000..e5b1139
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import static org.apache.datasketches.characterization.Shuffle.shuffle;
+
+import org.apache.datasketches.Properties;
+import org.apache.datasketches.kll.KllDoublesSketch;
+import org.apache.datasketches.kll.KllFloatsSketch;
+
+/**
+ * This handles either floats or doubles.
+ * @author Lee Rhodes
+ *
+ */
+public class KllSketchAccuracyProfile extends BaseQuantilesAccuracyProfile {
+
+ private int k;
+ private float[] inputFloatValues;
+ private float[] floatQueryValues;
+ private double[] inputDoubleValues;
+ private double[] doubleQueryValues;
+ private boolean useBulk;
+ private String type;
+ private boolean useFloat;
+ private boolean useDouble;
+ //add other types
+
+ @Override
+ void configure(final Properties props) {
+ useFloat = useDouble = false;
+ k = Integer.parseInt(props.mustGet("K"));
+ useBulk = Boolean.parseBoolean(props.mustGet("useBulk"));
+ type = props.mustGet("type");
+ if (type.equalsIgnoreCase("float")) { useFloat = true; }
+ else if (type.equalsIgnoreCase("double")) { useDouble = true; }
+ else { throw new IllegalArgumentException("Unknown primitive type: " + type); }
+ }
+
+ @Override
+ void prepareTrial(final int streamLength) {
+ // prepare input data that will be permuted
+ if (useFloat) {
+ inputFloatValues = new float[streamLength];
+ for (int i = 0; i < streamLength; i++) {
+ inputFloatValues[i] = i;
+ }
+ if (useBulk) {
+ // prepare query data that must remain ordered
+ floatQueryValues = new float[streamLength];
+ for (int i = 0; i < streamLength; i++) {
+ floatQueryValues[i] = i;
+ }
+ }
+ }
+ else if (useDouble) {
+ inputDoubleValues = new double[streamLength];
+ for (int i = 0; i < streamLength; i++) {
+ inputDoubleValues[i] = i;
+ }
+ if (useBulk) {
+ // prepare query data that must remain ordered
+ doubleQueryValues = new double[streamLength];
+ for (int i = 0; i < streamLength; i++) {
+ doubleQueryValues[i] = i;
+ }
+ }
+ }
+ }
+
+ @Override
+ double doTrial() {
+ double maxRankError = 0;
+ if (useFloat) {
+ shuffle(inputFloatValues);
+ // build sketch
+ final KllFloatsSketch sketch = new KllFloatsSketch(k);
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ sketch.update(inputFloatValues[i]);
+ }
+
+ // query sketch and gather results
+ maxRankError = 0;
+ if (useBulk) {
+ final double[] estRanks = sketch.getCDF(floatQueryValues);
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ final double trueRank = (double) i / inputFloatValues.length;
+ maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRanks[i]));
+ }
+ } else {
+ for (int i = 0; i < inputFloatValues.length; i++) {
+ final double trueRank = (double) i / inputFloatValues.length;
+ final double estRank = sketch.getRank(i);
+ maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRank));
+ }
+ }
+ }
+ else if (useDouble) {
+ shuffle(inputDoubleValues);
+ // build sketch
+ final KllDoublesSketch sketch = new KllDoublesSketch(k);
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ sketch.update(inputDoubleValues[i]);
+ }
+
+ // query sketch and gather results
+ maxRankError = 0;
+ if (useBulk) {
+ final double[] estRanks = sketch.getCDF(doubleQueryValues);
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ final double trueRank = (double) i / inputDoubleValues.length;
+ maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRanks[i]));
+ }
+ } else {
+ for (int i = 0; i < inputDoubleValues.length; i++) {
+ final double trueRank = (double) i / inputDoubleValues.length;
+ final double estRank = sketch.getRank(i);
+ maxRankError = Math.max(maxRankError, Math.abs(trueRank - estRank));
+ }
+ }
+ }
+ return maxRankError;
+ }
+
+}
diff --git a/src/main/resources/quantiles/KllFloatsSketchAccuracyJob.conf b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
similarity index 97%
rename from src/main/resources/quantiles/KllFloatsSketchAccuracyJob.conf
rename to src/main/resources/quantiles/KllSketchAccuracyJob.conf
index 6459759..beaf26e 100644
--- a/src/main/resources/quantiles/KllFloatsSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-JobProfile=org.apache.datasketches.characterization.quantiles.KllFloatsSketchAccuracyProfile
+JobProfile=org.apache.datasketches.characterization.quantiles.KllSketchAccuracyProfile
K=200 # sketch size and accuracy parameter
useBulk=true # query ranks using getCDF() instead of separate getRank() for each value
errLgK=10 # parameter for the rank error distribution sketch
@@ -24,3 +24,4 @@ trials=10 # Trials at every stream length
lgMin=0 # The starting stream length
lgMax=23 # How high the stream length goes
PPO=8 # The horizontal x-resolution of trials points
+type=float
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org