You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/22 12:08:15 UTC
incubator-kylin git commit: KYLIN-1068 add check in TopNAggregator
and DoubleDeltaSerializer
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1068 2fcb06e33 -> bf17d9d20
KYLIN-1068 add check in TopNAggregator and DoubleDeltaSerializer
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bf17d9d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bf17d9d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bf17d9d2
Branch: refs/heads/KYLIN-1068
Commit: bf17d9d20b6a8ec90b3dfb0797f69afd04c60989
Parents: 2fcb06e
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 22 18:07:57 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 22 18:07:57 2015 +0800
----------------------------------------------------------------------
.../common/topn/DoubleDeltaSerializer.java | 14 +++++++
.../kylin/metadata/measure/TopNAggregator.java | 39 +++++++++++++++++++-
2 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bf17d9d2/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
index cd798e0..f28aeea 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
@@ -58,6 +58,11 @@ public class DoubleDeltaSerializer {
}
public void serialize(double values[], ByteBuffer buf) {
+ double total = 0.0;
+ for (double value: values) {
+ total += value;
+ }
+ buf.putDouble(total);
long[] deltas = calculateDeltas(values);
int deltaSize = maxDeltaSize(deltas, values.length - 1);
@@ -137,11 +142,20 @@ public class DoubleDeltaSerializer {
}
public double[] deserialize(ByteBuffer buf) {
+ double expectedTotal = buf.getDouble();
int meta = buf.getInt();
int len = meta & ((1 << LENGTH_BITS) - 1);
double[] result = new double[len];
deserialize(buf, meta, result);
+ double actualTotal = 0.0;
+ for (double value : result) {
+ actualTotal += value;
+ }
+
+ if (Math.abs(actualTotal - expectedTotal) > 1.0) {
+ System.err.println("DoubleDeltaSerializer expected: " + expectedTotal + ", actual: " + actualTotal + ", array size:" + result.length + ", delta is: " + (expectedTotal - actualTotal));
+ }
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bf17d9d2/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
index 9567fe6..11e71f3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
@@ -18,9 +18,14 @@
package org.apache.kylin.metadata.measure;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
+import java.util.Iterator;
+import java.util.Map;
+
/**
*
*/
@@ -28,6 +33,7 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
int capacity = 0;
TopNCounter<ByteArray> sum = null;
+ Map<ByteArray, Double> sanityCheckMap;
@Override
public void reset() {
@@ -38,16 +44,45 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
public void aggregate(TopNCounter<ByteArray> value) {
if (sum == null) {
capacity = value.getCapacity();
- sum = new TopNCounter<ByteArray>(capacity);
+ sum = new TopNCounter<>(capacity);
+ sanityCheckMap = Maps.newHashMap();
}
-
+ addToSanityCheckMap(value);
sum.merge(value);
}
+
+ private void addToSanityCheckMap(TopNCounter<ByteArray> value) {
+
+ Iterator<Counter<ByteArray>> iterator = value.iterator();
+ while (iterator.hasNext()) {
+ Counter<ByteArray> counter = iterator.next();
+ if (sanityCheckMap.get(counter.getItem()) == null) {
+ sanityCheckMap.put(counter.getItem(), counter.getCount());
+ } else {
+ sanityCheckMap.put(counter.getItem(), counter.getCount() + sanityCheckMap.get(counter.getItem()));
+ }
+
+ }
+
+ }
+
+ private void sanityCheck() {
+ Iterator<Counter<ByteArray>> iterator = sum.iterator();
+ while (iterator.hasNext()) {
+ Counter<ByteArray> counter = iterator.next();
+ double expectedValue = sanityCheckMap.get(counter.getItem());
+ double actualValue = counter.getCount();
+ if (Math.abs(actualValue - expectedValue) > 1) {
+ throw new IllegalStateException("TopNAggregator sanity check failed, expected: " + expectedValue + ", actual: " + actualValue + ", key:" + counter.getItem() + ", delta is: " + (expectedValue - actualValue));
+ }
+ }
+ }
@Override
public TopNCounter getState() {
//sum.retain(capacity);
+ sanityCheck();
return sum;
}