You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/26 10:08:01 UTC
[15/45] incubator-kylin git commit: KYLIN-1048 fix CPU and memory
killer in Cuboid.findById()
KYLIN-1048 fix CPU and memory killer in Cuboid.findById()
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e82eb002
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e82eb002
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e82eb002
Branch: refs/heads/master
Commit: e82eb002f1b228999d5a18ebd928625773bb2b29
Parents: e924a2d
Author: shaofengshi <sh...@apache.org>
Authored: Mon Sep 28 15:30:52 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Sep 28 15:30:52 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/cuboid/Cuboid.java | 113 ++++++++++++++++---
1 file changed, 100 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e82eb002/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 93fa810..9080115 100644
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -18,23 +18,15 @@
package org.apache.kylin.cube.cuboid;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
+import org.apache.kylin.cube.model.*;
import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
/**
* @author George Song (ysong1)
*/
@@ -55,6 +47,10 @@ public class Cuboid implements Comparable<Cuboid> {
Cuboid cuboid = cubeCache.get(cuboidID);
if (cuboid == null) {
long validCuboidID = translateToValidCuboid(cube, cuboidID);
+ if (Cuboid.isValid(cube, validCuboidID) == false) {
+ throw new RuntimeException("Didn't find a valid cuboid: " + validCuboidID);
+ }
+
cuboid = new Cuboid(cube, cuboidID, validCuboidID);
cubeCache.put(cuboidID, cuboid);
}
@@ -92,8 +88,99 @@ public class Cuboid implements Comparable<Cuboid> {
return cube.getRowkey().getFullMask();
}
+ private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
+ // add mandantory
+ RowKeyDesc rowkey = cubeDesc.getRowkey();
+ long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
+ if (cuboidID < mandatoryColumnMask) {
+ cuboidID = cuboidID | mandatoryColumnMask;
+ }
+
+ // add hierarchy
+ for (DimensionDesc dimension : cubeDesc.getDimensions()) {
+ HierarchyDesc[] hierarchies = dimension.getHierarchy();
+ boolean found = false;
+ long result = 0;
+ if (hierarchies != null && hierarchies.length > 0) {
+ for (int i = hierarchies.length - 1; i >= 0; i--) {
+ TblColRef hColumn = hierarchies[i].getColumnRef();
+ Integer index = rowkey.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+
+ if ((rowkey.getTailMask() & bit) > 0)
+ continue; // ignore levels in tail, they don't participate
+
+ if ((bit & cuboidID) > 0) {
+ found = true;
+ }
+
+ if (found == true) {
+ result = result | bit;
+ }
+ }
+ cuboidID = cuboidID | result;
+ }
+ }
+
+ // find the left-most aggregation group
+ long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
+ long leftover;
+ for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
+ if ((cuboidWithoutMandatory & mask.uniqueMask) == mask.uniqueMask) {
+ leftover = cuboidWithoutMandatory & ~mask.groupMask;
+
+ if (leftover == 0) {
+ return cuboidID;
+ }
+
+ if (leftover != 0) {
+ cuboidID = cuboidID | mask.leftoverMask;
+ return cuboidID;
+ }
+ }
+ }
+
+ // doesn't have column in aggregation groups
+ leftover = cuboidWithoutMandatory & rowkey.getTailMask();
+ if (leftover == 0) {
+ // doesn't have column in tail group
+ if (cuboidWithoutMandatory != 0) {
+ return cuboidID;
+ } else {
+ // no column (except mandatory), add one column
+ long toAddCol = (1 << (BitSet.valueOf(new long[] { rowkey.getTailMask() }).cardinality()));
+ // check if the toAddCol belongs to any hierarchy
+ List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
+ if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
+ for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
+ long result = toAddCol & hierarchyMasks.fullMask;
+ if (result > 0) {
+ // replace it with the root col in hierarchy
+ toAddCol = hierarchyMasks.allMasks[0];
+ break;
+ }
+ }
+ }
+ cuboidID = cuboidID | toAddCol;
+ return cuboidID;
+ }
+ }
+
+ // has column in tail group
+ cuboidID = cuboidID | rowkey.getTailMask();
+ return cuboidID;
+
+ }
+
// Breadth-First-Search
- private static long translateToValidCuboid(CubeDesc cube, long cuboidID) {
+
+ /**
+ * @deprecated due to poor performance
+ * @param cube
+ * @param cuboidID
+ * @return
+ */
+ private static long translateToValidCuboidDeprecated(CubeDesc cube, long cuboidID) {
if (Cuboid.isValid(cube, cuboidID)) {
return cuboidID;
}