You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/22 12:08:15 UTC

incubator-kylin git commit: KYLIN-1068 add check in TopNAggregator and DoubleDeltaSerializer

Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-1068 2fcb06e33 -> bf17d9d20


KYLIN-1068 add check in TopNAggregator and DoubleDeltaSerializer

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bf17d9d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bf17d9d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bf17d9d2

Branch: refs/heads/KYLIN-1068
Commit: bf17d9d20b6a8ec90b3dfb0797f69afd04c60989
Parents: 2fcb06e
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 22 18:07:57 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 22 18:07:57 2015 +0800

----------------------------------------------------------------------
 .../common/topn/DoubleDeltaSerializer.java      | 14 +++++++
 .../kylin/metadata/measure/TopNAggregator.java  | 39 +++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bf17d9d2/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
index cd798e0..f28aeea 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
@@ -58,6 +58,11 @@ public class DoubleDeltaSerializer {
     }
 
     public void serialize(double values[], ByteBuffer buf) {
+        double total = 0.0;
+        for (double value: values) {
+            total += value;
+        }
+        buf.putDouble(total);
         long[] deltas = calculateDeltas(values);
         int deltaSize = maxDeltaSize(deltas, values.length - 1);
 
@@ -137,11 +142,20 @@ public class DoubleDeltaSerializer {
     }
 
     public double[] deserialize(ByteBuffer buf) {
+        double expectedTotal = buf.getDouble();
         int meta = buf.getInt();
         int len = meta & ((1 << LENGTH_BITS) - 1);
         
         double[] result = new double[len];
         deserialize(buf, meta, result);
+        double actualTotal = 0.0;
+        for (double value : result) {
+            actualTotal += value;
+        }
+        
+        if (Math.abs(actualTotal - expectedTotal) > 1.0) {
+            System.err.println("DoubleDeltaSerializer expected: " + expectedTotal + ", actual: " + actualTotal + ", array size:" + result.length + ", delta is: " + (expectedTotal - actualTotal));
+        }
         return result;
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bf17d9d2/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
index 9567fe6..11e71f3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
@@ -18,9 +18,14 @@
 
 package org.apache.kylin.metadata.measure;
 
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.topn.Counter;
 import org.apache.kylin.common.topn.TopNCounter;
 import org.apache.kylin.common.util.ByteArray;
 
+import java.util.Iterator;
+import java.util.Map;
+
 /**
  * 
  */
@@ -28,6 +33,7 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
 
     int capacity = 0;
     TopNCounter<ByteArray> sum = null;
+    Map<ByteArray, Double> sanityCheckMap;
 
     @Override
     public void reset() {
@@ -38,16 +44,45 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
     public void aggregate(TopNCounter<ByteArray> value) {
         if (sum == null) {
             capacity = value.getCapacity();
-            sum = new TopNCounter<ByteArray>(capacity);
+            sum = new TopNCounter<>(capacity);
+            sanityCheckMap = Maps.newHashMap();
         }
-
+        addToSanityCheckMap(value);
         sum.merge(value);
     }
+    
+    private void addToSanityCheckMap(TopNCounter<ByteArray> value) {
+
+        Iterator<Counter<ByteArray>> iterator = value.iterator();
+        while (iterator.hasNext()) {
+            Counter<ByteArray> counter = iterator.next();
+            if (sanityCheckMap.get(counter.getItem()) == null) {
+                sanityCheckMap.put(counter.getItem(), counter.getCount());
+            } else {
+                sanityCheckMap.put(counter.getItem(), counter.getCount() + sanityCheckMap.get(counter.getItem()));
+            }
+                
+        }
+        
+    }
+    
+    private void sanityCheck() {
+        Iterator<Counter<ByteArray>> iterator = sum.iterator();
+        while (iterator.hasNext()) {
+            Counter<ByteArray> counter = iterator.next();
+            double expectedValue = sanityCheckMap.get(counter.getItem());
+            double actualValue = counter.getCount();
+            if (Math.abs(actualValue - expectedValue) > 1) {
+                throw new IllegalStateException("TopNAggregator sanity check failed, expected: " + expectedValue + ", actual: " + actualValue + ", key:" + counter.getItem() + ", delta is: " + (expectedValue - actualValue));
+            }
+        }
+    }
 
     @Override
     public TopNCounter getState() {
         
         //sum.retain(capacity);
+        sanityCheck();
         return sum;
     }