You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/21 09:55:52 UTC
incubator-kylin git commit: KYLIN-1068 add sanity check for
TopNCounter
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1068 a5200d1bd -> 5b23d5612
KYLIN-1068 add sanity check for TopNCounter
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5b23d561
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5b23d561
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5b23d561
Branch: refs/heads/KYLIN-1068
Commit: 5b23d561215e941456796379a04bba759131ccae
Parents: a5200d1
Author: shaofengshi <sh...@apache.org>
Authored: Wed Oct 21 15:55:27 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Oct 21 15:55:27 2015 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/InMemCubeBuilder.java | 28 ++++++++++++++------
.../kylin/gridtable/GTAggregateScanner.java | 2 +-
.../serializer/TopNCounterSerializer.java | 4 +--
3 files changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b23d561/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index bc01caf..287b465 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -30,10 +30,9 @@ import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.topn.Counter;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.*;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
@@ -454,10 +453,10 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
mask = mask >> 1;
}
- return scanAndAggregateGridTable(parent.table, cuboidId, childDimensions, measureColumns);
+ return scanAndAggregateGridTable(parent.table, parent.cuboidId, cuboidId, childDimensions, measureColumns);
}
- private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException {
+ private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException {
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + cuboidId);
@@ -481,7 +480,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
// disable sanity check for performance
- sanityCheck(scanner.getTotalSumForSanityCheck());
+ sanityCheck(parentId, cuboidId, scanner.getTotalSumForSanityCheck());
} finally {
scanner.close();
builder.close();
@@ -494,12 +493,22 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
//@SuppressWarnings("unused")
- private void sanityCheck(Object[] totalSum) {
+ private void sanityCheck(long parentId, long cuboidId, Object[] totalSum) {
// double sum introduces error and causes result not exactly equal
for (int i = 0; i < totalSum.length; i++) {
if (totalSum[i] instanceof DoubleMutable) {
totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get());
+ } else if (totalSum[i] instanceof TopNCounter) {
+ TopNCounter counter = (TopNCounter) totalSum[i];
+ Iterator<Counter> iterator = counter.iterator();
+ double total = 0.0;
+ while (iterator.hasNext()) {
+ total += iterator.next().getCount();
+ }
+
+ totalSum[i] = Math.round(total);
}
+
}
if (totalSumForSanityCheck == null) {
@@ -507,6 +516,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
return;
}
if (Arrays.equals(totalSumForSanityCheck, totalSum) == false) {
+ logger.error("sanityCheck failed when calculate " + cuboidId + " from parent " + parentId);
+ logger.info("Expected: " + Arrays.toString(totalSumForSanityCheck));
+ logger.info("Actually: " + Arrays.toString(totalSum));
throw new IllegalStateException();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b23d561/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 9050c49..7c42086 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -167,7 +167,7 @@ public class GTAggregateScanner implements IGTScanner {
// skip expensive aggregation
for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator || totalSum[i] instanceof TopNAggregator )
+ if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator )
totalSum[i] = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b23d561/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
index f96dd00..f6cacde 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
@@ -34,7 +34,7 @@ import java.util.List;
*/
public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteArray>> {
- private DoubleDeltaSerializer dds = new DoubleDeltaSerializer();
+ private DoubleDeltaSerializer dds = new DoubleDeltaSerializer(3);
private int precision;
@@ -98,7 +98,7 @@ public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteAr
TopNCounter<ByteArray> counter = new TopNCounter<ByteArray>(capacity);
ByteArray byteArray;
- for(int i=0; i<size; i++) {
+ for (int i = 0; i < size; i++) {
byteArray = new ByteArray(keyLength);
in.get(byteArray.array());
counter.offerToHead(byteArray, counters[i]);