You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/28 07:48:46 UTC
kylin git commit: KYLIN-1635 impl was done, this is a bug fix
Repository: kylin
Updated Branches:
refs/heads/yang-m1 3d54189dc -> 71a009e75
KYLIN-1635 impl was done, this is a bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71a009e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71a009e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71a009e7
Branch: refs/heads/yang-m1
Commit: 71a009e751420ca798c436d0eb7a9ef7a0107c07
Parents: 3d54189
Author: Yang Li <li...@apache.org>
Authored: Thu Apr 28 13:48:32 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Apr 28 13:48:32 2016 +0800
----------------------------------------------------------------------
.../measure/hllc/HyperLogLogPlusCounter.java | 1 +
.../measure/hll/HyperLogLogCounterTest.java | 39 ++++++++++++--------
2 files changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/71a009e7/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
index 4f9755b..29f7ffc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
@@ -246,6 +246,7 @@ public class HyperLogLogPlusCounter implements Serializable, Comparable<HyperLog
} else if (scheme == 1) { // array scheme
in.get(registers);
+ singleBucket = Integer.MIN_VALUE;
} else
throw new IllegalStateException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/71a009e7/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
index 15a3a47..ffd147d 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
@@ -149,32 +149,31 @@ public class HyperLogLogCounterTest {
hllc.readRegisters(buf);
Assert.assertEquals(estimate, hllc.getCountEstimate());
}
-
+
@Test
public void mergeTest() throws IOException {
double error = 0;
double absError = 0;
int n = 100;
for (int i = 0; i < n; i++) {
- double e = merge();
+ double e = merge(i);
error += e;
absError += Math.abs(e);
}
- System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n);
+ System.out.println("Total average error is " + error / n);
System.out.println(" errorRateCount1 is " + errorCount1 + "!");
System.out.println(" errorRateCount2 is " + errorCount2 + "!");
System.out.println(" errorRateCount3 is " + errorCount3 + "!");
- Assert.assertTrue(errorCount1 <= n * 0.40);
- Assert.assertTrue(errorCount2 <= n * 0.08);
+ Assert.assertTrue(errorCount1 <= n * 0.30);
+ Assert.assertTrue(errorCount2 <= n * 0.05);
Assert.assertTrue(errorCount3 <= n * 0.02);
}
- private double merge() throws IOException {
-
- int ln = 50;
- int dn = 300;
+ private double merge(int round) throws IOException {
+ int ln = 20;
+ int dn = 100 * (round + 1);
Set<String> testSet = new HashSet<String>();
HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln];
for (int i = 0; i < ln; i++) {
@@ -189,29 +188,39 @@ public class HyperLogLogCounterTest {
}
HyperLogLogPlusCounter mergeHllc = newHLLC();
for (HyperLogLogPlusCounter hllc : hllcs) {
- mergeHllc.merge(hllc);
- checkSerialize(mergeHllc);
+ mergeHllc.merge(serDes(hllc));
}
double errorRate = mergeHllc.getErrorRate();
long estimate = mergeHllc.getCountEstimate();
- double actualError = (double) (testSet.size() - estimate) / testSet.size();
+ double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
+ Assert.assertTrue(actualError < 0.1);
- if (Math.abs(actualError) > errorRate) {
+ if (actualError > errorRate) {
errorCount1++;
}
- if (Math.abs(actualError) > 2 * errorRate) {
+ if (actualError > 2 * errorRate) {
errorCount2++;
}
- if (Math.abs(actualError) > 3 * errorRate) {
+ if (actualError > 3 * errorRate) {
errorCount3++;
}
return actualError;
}
+ private HyperLogLogPlusCounter serDes(HyperLogLogPlusCounter hllc) throws IOException {
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ HyperLogLogPlusCounter copy = new HyperLogLogPlusCounter(hllc.getPrecision());
+ copy.readRegisters(buf);
+ Assert.assertEquals(copy.getCountEstimate(), hllc.getCountEstimate());
+ return copy;
+ }
+
@Test
public void testPerformance() throws IOException {
int N = 3; // reduce N HLLC into one