You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2019/11/11 22:57:19 UTC
[incubator-datasketches-characterization] 01/01: Characterization
tests for ZetaSketch,
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch ZetaTesting
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-characterization.git
commit b19a49067cb71100f3a87762447a4bbe5b5879e3
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Nov 11 14:56:34 2019 -0800
Characterization tests for ZetaSketch,
New BaseMergeSpeedProfile for standardized merge testing.
---
pom.xml | 8 ++
.../characterization/hll/HllMergeSpeedProfile.java | 143 +++++++++++++++++++
.../hll/ZetaHllAccuracyProfile.java | 84 +++++++++++
.../hll/ZetaHllMergeSpeedProfile.java | 89 ++++++++++++
.../characterization/hll/ZetaHllSerDeProfile.java | 110 +++++++++++++++
.../hll/ZetaHllUpdateSpeedProfile.java | 80 +++++++++++
.../uniquecount/BaseMergeSpeedProfile.java | 154 +++++++++++++++++++++
.../uniquecount/BaseSerDeProfile.java | 2 +
.../hll/zetasketchHll/ZetaHllAccuracyJob.conf | 46 ++++++
.../hll/zetasketchHll/ZetaHllMergeSpeedJob.conf | 39 ++++++
.../hll/zetasketchHll/ZetaHllSerDeJob.conf | 43 ++++++
.../hll/zetasketchHll/ZetaHllSpeedJob.conf | 43 ++++++
12 files changed, 841 insertions(+)
diff --git a/pom.xml b/pom.xml
index 7812611..0666b2e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,7 @@ under the License.
<druid-momentsketch.version>0.16.0-incubating</druid-momentsketch.version>
<druid-histogram.version>0.16.0-incubating</druid-histogram.version>
<druid-hll.version>0.16.0-incubating</druid-hll.version>
+ <zetasketch.version>0.1.0</zetasketch.version>
<!-- END:UNIQUE FOR THIS JAVA COMPONENT -->
<!-- Test -->
@@ -197,6 +198,13 @@ under the License.
<artifactId>druid-histogram</artifactId>
<version>${druid-histogram.version}</version>
</dependency>
+
+ <!-- ZetaSketch -->
+ <dependency>
+ <groupId>com.google.zetasketch</groupId>
+ <artifactId>zetasketch</artifactId>
+ <version>${zetasketch.version}</version>
+ </dependency>
<!-- Dependency on Test code -->
<dependency>
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/HllMergeSpeedProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/HllMergeSpeedProfile.java
new file mode 100644
index 0000000..b16349c
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/HllMergeSpeedProfile.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.characterization.uniquecount.BaseMergeSpeedProfile;
+import org.apache.datasketches.hll.HllSketch;
+import org.apache.datasketches.hll.TgtHllType;
+import org.apache.datasketches.hll.Union;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+
+/**
+ * @author Lee Rhodes
+ */
+public class HllMergeSpeedProfile extends BaseMergeSpeedProfile {
+ private boolean compact;
+ private boolean wrap;
+ private boolean direct;
+ private TgtHllType tgtHllType;
+ private Union union = new Union(21);
+
+ @Override
+ public void configure() {
+ direct = Boolean.parseBoolean(prop.mustGet("HLL_direct"));
+ compact = Boolean.parseBoolean(prop.mustGet("HLL_compact"));
+ wrap = Boolean.parseBoolean(prop.mustGet("HLL_wrap"));
+
+ final String type = prop.mustGet("HLL_tgtHllType");
+ if (type.equalsIgnoreCase("HLL4")) { tgtHllType = TgtHllType.HLL_4; }
+ else if (type.equalsIgnoreCase("HLL6")) { tgtHllType = TgtHllType.HLL_6; }
+ else { tgtHllType = TgtHllType.HLL_8; }
+ }
+
+ @Override
+ public void resetMerge(final int lgK) {
+ union = new Union(lgK);
+ }
+
+ private HllSketch newSketch(final int lgK) {
+ WritableMemory wmem = null;
+ final HllSketch sk;
+ if (direct) {
+ final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+ wmem = WritableMemory.allocate(bytes);
+ sk = new HllSketch(lgK, tgtHllType, wmem);
+ } else {
+ sk = new HllSketch(lgK, tgtHllType);
+ }
+ return sk;
+ }
+
+
+ @Override
+ public void doTrial(final Stats stats, final int lgK, final int lgDeltaU) {
+ final int U = 1 << (lgK + lgDeltaU);
+ long start;
+ long serTime_nS = 0;
+ long deserTime_nS = 0;
+ long mergeTime_nS = 0;
+ final HllSketch source = newSketch(lgK);
+ final long vStartUnion = vIn;
+
+ final long vStart = vIn;
+ source.reset();
+ for (int u = 0; u < U; u++) { source.update(++vIn); }
+ final long trueU = vIn - vStart;
+ //checkEstimate(trueU, source.getEstimate(), lgK, "Source");
+ HllSketch source2 = null;
+ final byte[] byteArr;
+
+ if (serDe) {
+ //Serialize
+ if (compact) {
+ start = System.nanoTime();
+ byteArr = source.toCompactByteArray();
+ serTime_nS += System.nanoTime() - start;
+ } else {
+ start = System.nanoTime();
+ byteArr = source.toUpdatableByteArray();
+ serTime_nS += System.nanoTime() - start;
+ }
+ //Deserialize
+ if (wrap) {
+ start = System.nanoTime();
+ final Memory mem = Memory.wrap(byteArr);
+ source2 = HllSketch.wrap(mem);
+ deserTime_nS += System.nanoTime() - start;
+ } else { //heapify
+ start = System.nanoTime();
+ final Memory mem = Memory.wrap(byteArr);
+ source2 = HllSketch.heapify(mem);
+ deserTime_nS += System.nanoTime() - start;
+ }
+ //checkEstimate(trueU, source2.getEstimate(), lgK, "SerDe");
+ //Merge
+ start = System.nanoTime();
+ union.update(source2);
+ mergeTime_nS += System.nanoTime() - start;
+
+ } else {
+ //Merge
+ start = System.nanoTime();
+ union.update(source);
+ mergeTime_nS += System.nanoTime() - start;
+ }
+
+ stats.serializeTime_nS = serTime_nS;
+ stats.deserializeTime_nS = deserTime_nS;
+ stats.mergeTime_nS = mergeTime_nS;
+ stats.totalTime_nS = mergeTime_nS;
+
+ //final double vUnionActual = vIn - vStartUnion;
+ //checkEstimate(vUnionActual, union.getEstimate(), lgK, "Union");
+
+ }
+
+ void checkEstimate(final double actual, final double est, final int lgK, final String note) {
+ final double k = 1L << lgK;
+ final double bound = 3.0 / Math.sqrt(k);
+ final double err = Math.abs((est / actual) - 1.0);
+ if (err > bound) {
+ System.out.printf("ERROR: %12.3f %12.3f %20s\n", err, bound, note);
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java
new file mode 100644
index 0000000..0e06ae4
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllAccuracyProfile.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.AccuracyStats;
+import org.apache.datasketches.characterization.uniquecount.BaseAccuracyProfile;
+
+import com.google.zetasketch.HyperLogLogPlusPlus;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ZetaHllAccuracyProfile extends BaseAccuracyProfile {
+ private enum ZetaType { LONG, INTEGER, STRING, BYTES }
+
+ private HyperLogLogPlusPlus<?> sketch;
+ private HyperLogLogPlusPlus.Builder hllBuilder;
+ private int lgSP;
+ private String zetaType;
+ private ZetaType zType;
+
+ @Override
+ public void configure() {
+ lgSP = Integer.parseInt(prop.mustGet("LgSP"));
+ zetaType = prop.mustGet("ZetaType");
+ hllBuilder = new HyperLogLogPlusPlus.Builder();
+ hllBuilder.normalPrecision(lgK);
+ hllBuilder.sparsePrecision(lgSP);
+ if (zetaType.equals("LONG")) {
+ zType = ZetaType.LONG;
+ } else if (zetaType.equals("INTEGER")) {
+ zType = ZetaType.INTEGER;
+ } else if (zetaType.equals("STRING")) {
+ zType = ZetaType.STRING;
+ } else if (zetaType.equals("BYTES")) {
+ zType = ZetaType.BYTES;
+ }
+ reset();
+ }
+
+ private void reset() {
+ switch (zType) {
+ case LONG: sketch = hllBuilder.buildForLongs(); break;
+ case INTEGER: sketch = hllBuilder.buildForIntegers(); break;
+ case STRING: sketch = hllBuilder.buildForStrings(); break;
+ case BYTES: sketch = hllBuilder.buildForBytes(); break;
+ }
+ }
+
+ @Override
+ public void doTrial() {
+ final int qArrLen = qArr.length;
+ reset();
+ int lastUniques = 0;
+ for (int i = 0; i < qArrLen; i++) {
+ final AccuracyStats q = qArr[i];
+ final double delta = q.trueValue - lastUniques;
+ for (int u = 0; u < delta; u++) {
+ sketch.add(++vIn);
+ }
+ lastUniques += delta;
+ final double est = sketch.result();
+ q.update(est);
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllMergeSpeedProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllMergeSpeedProfile.java
new file mode 100644
index 0000000..4dc4eff
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllMergeSpeedProfile.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.characterization.uniquecount.BaseMergeSpeedProfile;
+
+import com.google.zetasketch.HyperLogLogPlusPlus;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ZetaHllMergeSpeedProfile extends BaseMergeSpeedProfile {
+ private HyperLogLogPlusPlus.Builder hllBuilder;
+ private HyperLogLogPlusPlus<Long> target;
+
+ @Override
+ public void configure() {
+ hllBuilder = new HyperLogLogPlusPlus.Builder();
+ }
+
+ @Override
+ public void resetMerge(final int lgK) {
+ target = newSketch(lgK);
+ }
+
+ private HyperLogLogPlusPlus<Long> newSketch(final int lgK) {
+ final int lgSP = Math.min(lgK + 5, 25);
+ hllBuilder.normalPrecision(lgK);
+ hllBuilder.sparsePrecision(lgSP);
+ return hllBuilder.buildForLongs();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void doTrial(final Stats stats, final int lgK, final int lgDeltaU) {
+ final int U = 1 << (lgK + lgDeltaU);
+ long start;
+ long serTime_nS = 0;
+ long deserTime_nS = 0;
+ long mergeTime_nS = 0;
+ final HyperLogLogPlusPlus<Long> source = newSketch(lgK);
+ for (int u = 0; u < U; u++) { source.add(++vIn); }
+ final HyperLogLogPlusPlus<Long> source2;
+
+ if (serDe) {
+ //Serialize
+ start = System.nanoTime();
+ final byte[] byteArr = source.serializeToByteArray();
+ serTime_nS += System.nanoTime() - start;
+ //Deserialize
+ start = System.nanoTime();
+ source2 = (HyperLogLogPlusPlus<Long>) HyperLogLogPlusPlus.forProto(byteArr);
+ deserTime_nS += System.nanoTime() - start;
+ //Merge
+ start = System.nanoTime();
+ target.merge(source2);
+ mergeTime_nS += System.nanoTime() - start;
+
+ } else {
+ //Merge
+ start = System.nanoTime();
+ target.merge(source);
+ mergeTime_nS += System.nanoTime() - start;
+ }
+
+ stats.serializeTime_nS = serTime_nS;
+ stats.deserializeTime_nS = deserTime_nS;
+ stats.mergeTime_nS = mergeTime_nS;
+ stats.totalTime_nS = deserTime_nS + mergeTime_nS;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllSerDeProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllSerDeProfile.java
new file mode 100644
index 0000000..27f9b63
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllSerDeProfile.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.characterization.uniquecount.BaseSerDeProfile;
+
+import com.google.zetasketch.HyperLogLogPlusPlus;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ZetaHllSerDeProfile extends BaseSerDeProfile {
+ private enum ZetaType { LONG, INTEGER, STRING, BYTES }
+
+ private HyperLogLogPlusPlus<?> sketch1;
+ private HyperLogLogPlusPlus<?> sketch2;
+ private HyperLogLogPlusPlus.Builder hllBuilder;
+
+ private int lgSP;
+ private String zetaType;
+ private ZetaType zType;
+
+ @Override
+ public void configure() {
+ lgSP = Integer.parseInt(prop.mustGet("LgSP"));
+ zetaType = prop.mustGet("ZetaType");
+ hllBuilder = new HyperLogLogPlusPlus.Builder();
+ hllBuilder.normalPrecision(lgK);
+ hllBuilder.sparsePrecision(lgSP);
+ if (zetaType.equals("LONG")) {
+ zType = ZetaType.LONG;
+ } else if (zetaType.equals("INTEGER")) {
+ zType = ZetaType.INTEGER;
+ } else if (zetaType.equals("STRING")) {
+ zType = ZetaType.STRING;
+ } else if (zetaType.equals("BYTES")) {
+ zType = ZetaType.BYTES;
+ }
+ reset();
+ }
+
+ private void reset() {
+ switch (zType) {
+ case LONG:
+ sketch1 = hllBuilder.buildForLongs();
+ sketch2 = hllBuilder.buildForLongs();
+ break;
+ case INTEGER:
+ sketch1 = hllBuilder.buildForIntegers();
+ sketch2 = hllBuilder.buildForIntegers();
+ break;
+ case STRING:
+ sketch1 = hllBuilder.buildForStrings();
+ sketch2 = hllBuilder.buildForStrings();
+ break;
+ case BYTES:
+ sketch1 = hllBuilder.buildForBytes();
+ sketch2 = hllBuilder.buildForBytes();
+ break;
+ }
+ }
+
+ @Override
+ public void doTrial(final Stats stats, final int uPerTrial) {
+ reset();
+ //Serialize
+ for (int u = uPerTrial; u-- > 0;) {
+ sketch1.add(++vIn);
+ }
+ final double est1 = sketch1.result();
+
+ final byte[] byteArr;
+ final long startSerTime_nS, stopSerTime_nS;
+
+ startSerTime_nS = System.nanoTime();
+ byteArr = sketch1.serializeToByteArray();
+ stopSerTime_nS = System.nanoTime();
+
+ //Deserialize
+ final long startDeserTime_nS, stopDeserTime_nS;
+ startDeserTime_nS = System.nanoTime();
+ sketch2 = HyperLogLogPlusPlus.forProto(byteArr);
+ stopDeserTime_nS = System.nanoTime();
+
+ final double est2 = sketch2.result();
+ assert est1 == est2;
+
+ stats.serializeTime_nS = stopSerTime_nS - startSerTime_nS;
+ stats.deserializeTime_nS = stopDeserTime_nS - startDeserTime_nS;
+ stats.size_bytes = byteArr.length;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllUpdateSpeedProfile.java b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllUpdateSpeedProfile.java
new file mode 100644
index 0000000..14b730c
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/hll/ZetaHllUpdateSpeedProfile.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.hll;
+
+import org.apache.datasketches.characterization.uniquecount.BaseUpdateSpeedProfile;
+
+import com.google.zetasketch.HyperLogLogPlusPlus;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ZetaHllUpdateSpeedProfile extends BaseUpdateSpeedProfile {
+ private enum ZetaType { LONG, INTEGER, STRING, BYTES }
+
+ private HyperLogLogPlusPlus<?> sketch;
+ private HyperLogLogPlusPlus.Builder hllBuilder;
+ private int lgK;
+ private int lgSP;
+ private String zetaType;
+ private ZetaType zType;
+
+ @Override
+ public void configure() {
+ lgK = Integer.parseInt(prop.mustGet("LgK"));
+ lgSP = Integer.parseInt(prop.mustGet("LgSP"));
+ zetaType = prop.mustGet("ZetaType");
+ hllBuilder = new HyperLogLogPlusPlus.Builder();
+ hllBuilder.normalPrecision(lgK);
+ hllBuilder.sparsePrecision(lgSP);
+ if (zetaType.equals("LONG")) {
+ zType = ZetaType.LONG;
+ } else if (zetaType.equals("INTEGER")) {
+ zType = ZetaType.INTEGER;
+ } else if (zetaType.equals("STRING")) {
+ zType = ZetaType.STRING;
+ } else if (zetaType.equals("BYTES")) {
+ zType = ZetaType.BYTES;
+ }
+ reset();
+ }
+
+ private void reset() {
+ switch (zType) {
+ case LONG: sketch = hllBuilder.buildForLongs(); break;
+ case INTEGER: sketch = hllBuilder.buildForIntegers(); break;
+ case STRING: sketch = hllBuilder.buildForStrings(); break;
+ case BYTES: sketch = hllBuilder.buildForBytes(); break;
+ }
+ }
+
+ @Override
+ public double doTrial(final int uPerTrial) {
+ reset();
+ final long startUpdateTime_nS = System.nanoTime();
+
+ for (int u = uPerTrial; u-- > 0;) {
+ sketch.add(++vIn);
+ }
+ final long updateTime_nS = System.nanoTime() - startUpdateTime_nS;
+ return (double) updateTime_nS / uPerTrial;
+ }
+
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseMergeSpeedProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseMergeSpeedProfile.java
new file mode 100644
index 0000000..7d22f6b
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseMergeSpeedProfile.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.uniquecount;
+
+//import static java.lang.Math.log;
+//import static java.lang.Math.pow;
+
+import org.apache.datasketches.Job;
+import org.apache.datasketches.JobProfile;
+import org.apache.datasketches.Properties;
+
+/**
+ * @author Lee Rhodes
+ */
+//@SuppressWarnings("javadoc")
+public abstract class BaseMergeSpeedProfile implements JobProfile {
+ Job job;
+ public Properties prop;
+ public long vIn = 0;
+ int minLgT;
+ int maxLgT;
+ int minLgK;
+ int maxLgK;
+ public int lgDeltaU;
+ public boolean serDe;
+
+ public Stats stats = new Stats();
+
+ //JobProfile
+ @Override
+ public void start(final Job job) {
+ this.job = job;
+ prop = job.getProperties();
+ minLgT = Integer.parseInt(prop.mustGet("MinLgT"));
+ maxLgT = Integer.parseInt(prop.mustGet("MaxLgT"));
+ minLgK = Integer.parseInt(prop.mustGet("MinLgK"));
+ maxLgK = Integer.parseInt(prop.mustGet("MaxLgK"));
+ lgDeltaU = Integer.parseInt(prop.mustGet("LgDeltaU"));
+ serDe = Boolean.parseBoolean(prop.mustGet("SerDe"));
+ configure();
+ doTrials();
+ shutdown();
+ cleanup();
+ }
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public void cleanup() {}
+
+ @Override
+ public void println(final Object obj) {
+ job.println(obj);
+ }
+ //end JobProfile
+
+ /**
+ * Configure the sketch
+ */
+ public abstract void configure();
+
+ /**
+ * Perform a single trial
+ * @param stats stats array
+ * @param lgK sketch size
+ * @param lgDeltaU delta size determining U: +1 = 2K, +2 = 4K; -1 = K/2, -2 = K/4, etc.
+ */
+ public abstract void doTrial(Stats stats, int lgK, int lgDeltaU);
+
+ public abstract void resetMerge(int lgK);
+
+ private void doTrials() {
+ final StringBuilder dataStr = new StringBuilder();
+ println(getHeader());
+ final Stats stats = new Stats();
+ int lgK;
+
+ for (lgK = minLgK; lgK <= maxLgK; lgK++) {
+ final int lgT = (maxLgK - lgK) + minLgT;
+ final int trials = 1 << lgT;
+ double sumSerializeTime_nS = 0;
+ double sumDeserialzeTime_nS = 0;
+ double sumMergeTime_nS = 0;
+ double sumTotalTime_nS = 0;
+ resetMerge(lgK);
+ for (int t = 0; t < trials; t++) {
+ doTrial(stats, lgK, lgDeltaU);
+ sumSerializeTime_nS += stats.serializeTime_nS;
+ sumDeserialzeTime_nS += stats.deserializeTime_nS;
+ sumMergeTime_nS += stats.mergeTime_nS;
+ sumTotalTime_nS += stats.totalTime_nS;
+ }
+ //Per sketch per trial
+ stats.serializeTime_nS = sumSerializeTime_nS / trials;
+ stats.deserializeTime_nS = sumDeserialzeTime_nS / trials;
+ stats.mergeTime_nS = sumMergeTime_nS / trials;
+ stats.totalTime_nS = sumTotalTime_nS / trials;
+ process(stats, lgK, lgT, dataStr);
+ println(dataStr.toString());
+ }
+ }
+
+ private static void process(final Stats stats,
+ final int lgK, final int lgT, final StringBuilder dataStr) {
+
+ //OUTPUT
+ dataStr.setLength(0);
+ dataStr.append(lgK).append(TAB);
+ dataStr.append(lgT).append(TAB);
+ dataStr.append(stats.serializeTime_nS).append(TAB);
+ dataStr.append(stats.deserializeTime_nS).append(TAB);
+ dataStr.append(stats.mergeTime_nS).append(TAB);
+ dataStr.append(stats.totalTime_nS).append(TAB);
+ final double slotTime_nS = stats.totalTime_nS / (1 << lgK);
+ dataStr.append(slotTime_nS);
+ }
+
+ private static String getHeader() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("LgK").append(TAB);
+ sb.append("LgT").append(TAB);
+ sb.append("Ser_nS").append(TAB);
+ sb.append("DeSer_nS").append(TAB);
+ sb.append("Merge_nS").append(TAB);
+ sb.append("Total_nS").append(TAB);
+ sb.append("PerSlot_nS");
+ return sb.toString();
+ }
+
+ public static class Stats {
+ public double serializeTime_nS;
+ public double deserializeTime_nS;
+ public double mergeTime_nS;
+ public double totalTime_nS;
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseSerDeProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseSerDeProfile.java
index a97fdaf..fa77183 100644
--- a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseSerDeProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseSerDeProfile.java
@@ -61,6 +61,8 @@ public abstract class BaseSerDeProfile implements JobProfile {
lgK = Integer.parseInt(prop.mustGet("LgK"));
configure();
doTrials();
+ shutdown();
+ cleanup();
}
@Override
diff --git a/src/main/resources/hll/zetasketchHll/ZetaHllAccuracyJob.conf b/src/main/resources/hll/zetasketchHll/ZetaHllAccuracyJob.conf
new file mode 100644
index 0000000..b26ef1c
--- /dev/null
+++ b/src/main/resources/hll/zetasketchHll/ZetaHllAccuracyJob.conf
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+
+#Uniques Profile
+Trials_lgMinU=0 #The starting # of uniques that is printed at the end.
+Trials_lgMaxU=23 #How high the # uniques go
+Trials_UPPO=16 #The horizontal x-resolution of trials points
+Trials_string=false #Used in Druid HLL profile
+
+# Trials Profile
+Trials_lgMinT=8 #prints intermediate results starting w/ this lgMinT
+Trials_lgMaxT=16 #The max trials
+Trials_TPPO=1 #how often intermediate results are printed
+
+Trials_lgQK=12 #size of quantiles sketch for analysis
+Trials_interData=true
+Trials_postPMFs=false
+Trials_bytes=false
+
+# Date-Time Profile
+TimeZone=PST
+TimeZoneOffset=-28800000 # offset in millisec
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
+
+#Job Profile
+JobProfile=org.apache.datasketches.characterization.hll.ZetaHllAccuracyProfile
+LgK=12 #Normal precision, Min 10, Max 24
+LgSP=17 #Sparse precision= Normal Prcision + 0 to 5, Max 25
+ZetaType=LONG #one of LONG,INTEGER,STRING,BYTES
diff --git a/src/main/resources/hll/zetasketchHll/ZetaHllMergeSpeedJob.conf b/src/main/resources/hll/zetasketchHll/ZetaHllMergeSpeedJob.conf
new file mode 100644
index 0000000..ef979ce
--- /dev/null
+++ b/src/main/resources/hll/zetasketchHll/ZetaHllMergeSpeedJob.conf
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+
+# Date-Time Profile
+TimeZone=PST
+TimeZoneOffset=-28800000 # offset in millisec
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
+
+#X-axis LgK Profile
+MinLgK=10
+MaxLgK=21
+
+# Trials Profile
+MinLgT=6 #Min Log Trails
+MaxLgT=6 #Max Log Trials
+
+#Job Profile
+JobProfile=org.apache.datasketches.characterization.hll.ZetaHllMergeSpeedProfile
+ZetaType=LONG #one of LONG,INTEGER,STRING,BYTES
+SerDe=true
+LgDeltaU=1
+
diff --git a/src/main/resources/hll/zetasketchHll/ZetaHllSerDeJob.conf b/src/main/resources/hll/zetasketchHll/ZetaHllSerDeJob.conf
new file mode 100644
index 0000000..1d36642
--- /dev/null
+++ b/src/main/resources/hll/zetasketchHll/ZetaHllSerDeJob.conf
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+
+#Uniques Profile
+Trials_lgMinU=0 #The starting # of uniques that is printed at the end.
+Trials_lgMaxU=24 #How high the # uniques go
+Trials_UPPO=2 #The horizontal x-resolution of trial points, Points Per Octave
+
+# Trials Profile
+Trials_lgMaxT=20 #Max trials at start (low counts)
+Trials_lgMinT=7 #Min trials at tail (high counts)
+
+#Trails Speed related
+Trials_lgMinBpU=4 #start the downward slope of trials at this LgU
+Trials_lgMaxBpU=20 #stop the downward slope of trials at this LgU
+
+# Date-Time Profile
+TimeZone=PST
+TimeZoneOffset=-28800000 # offset in millisec
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
+
+#Job Profile
+JobProfile=org.apache.datasketches.characterization.hll.ZetaHllSerDeProfile
+LgK=20 #Normal precision, Min 10, Max 24
+LgSP=25 #Sparse precision= Normal Prcision + 0 to 5, Max 25
+ZetaType=LONG #one of LONG,INTEGER,STRING,BYTES
\ No newline at end of file
diff --git a/src/main/resources/hll/zetasketchHll/ZetaHllSpeedJob.conf b/src/main/resources/hll/zetasketchHll/ZetaHllSpeedJob.conf
new file mode 100644
index 0000000..9b32341
--- /dev/null
+++ b/src/main/resources/hll/zetasketchHll/ZetaHllSpeedJob.conf
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+
+#Uniques Profile
+Trials_lgMinU=0 #The starting # of uniques that is printed at the end.
+Trials_lgMaxU=23 #How high the # uniques go
+Trials_UPPO=16 #The horizontal x-resolution of trials points
+
+# Trials Profile
+Trials_lgMaxT=24 #Max trials at start (low counts)
+Trials_lgMinT=4 #Min trials at tail (high counts)
+
+#Trails Speed related
+Trials_lgMinBpU=4 #start the downward slope of trials at this LgU
+Trials_lgMaxBpU=20 #stop the downward slope of trials at this LgU
+
+# Date-Time Profile
+TimeZone=PST
+TimeZoneOffset=-28800000 # offset in millisec
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss z
+
+#Job Profile
+JobProfile=org.apache.datasketches.characterization.hll.ZetaHllUpdateSpeedProfile
+LgK=12 #Normal precision, Min 10, Max 24
+LgSP=17 #Sparse precision= Normal Prcision + 0 to 5, Max 25
+ZetaType=LONG #one of LONG,INTEGER,STRING,BYTES
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org