You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/09 13:09:36 UTC
[2/4] kylin git commit: KYLIN-2783 Refactor CuboidScheduler to be
extensible
KYLIN-2783 Refactor CuboidScheduler to be extensible
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/12af6533
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/12af6533
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/12af6533
Branch: refs/heads/pr69
Commit: 12af6533ca3a9e8f812426ca9bb4fc29785cf121
Parents: 7bdce5e
Author: Zhong <nj...@apache.org>
Authored: Thu Aug 17 20:17:26 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 9 16:37:20 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeDescManager.java | 3 -
.../org/apache/kylin/cube/CubeInstance.java | 18 +-
.../java/org/apache/kylin/cube/CubeManager.java | 17 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 5 +
.../kylin/cube/common/RowKeySplitter.java | 4 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 180 +++--------
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 20 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 25 +-
.../cube/cuboid/DefaultCuboidScheduler.java | 162 +++++++++-
.../kylin/cube/cuboid/TreeCuboidScheduler.java | 322 +++++++++++++++++++
.../inmemcubing/AbstractInMemCubeBuilder.java | 17 +-
.../cube/inmemcubing/DoggedCubeBuilder.java | 9 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 11 +-
.../org/apache/kylin/cube/kv/RowKeyDecoder.java | 4 +-
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 11 +-
.../kylin/cube/model/AggregationGroup.java | 4 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 33 +-
.../cube/model/CubeJoinedFlatTableEnrich.java | 3 +-
.../org/apache/kylin/cube/util/CubingUtils.java | 2 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 26 +-
.../apache/kylin/cube/cuboid/CuboidTest.java | 60 ++--
.../apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +-
.../apache/kylin/cube/kv/RowKeyEncoderTest.java | 6 +-
.../gtrecord/GTCubeStorageQueryBase.java | 6 +-
.../kylin/storage/translate/HBaseKeyRange.java | 2 +-
.../kylin/engine/mr/BatchCubingJobBuilder.java | 2 +-
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 2 +-
.../engine/mr/common/BaseCuboidBuilder.java | 14 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 7 +-
.../mr/steps/FactDistinctColumnsMapper.java | 2 +-
.../engine/mr/steps/InMemCuboidMapper.java | 3 +-
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../kylin/engine/mr/steps/NDCuboidMapper.java | 6 +-
.../apache/kylin/engine/spark/SparkCubing.java | 9 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 12 +-
.../spark/cube/DefaultTupleConverter.java | 2 +-
.../ITDoggedCubeBuilderStressTest.java | 2 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 4 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 2 +-
.../storage/hbase/steps/HBaseCuboidWriter.java | 4 +-
.../cube/MeasureTypeOnlyAggrInBaseTest.java | 4 +-
42 files changed, 738 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 6635366..f49c9be 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -171,7 +171,6 @@ public class CubeDescManager {
CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false);
cubeDescMap.putLocal(ndesc.getName(), ndesc);
- Cuboid.reloadCache(name);
// if related cube is in DESCBROKEN state before, change it back to DISABLED
CubeManager cubeManager = CubeManager.getInstance(config);
@@ -297,13 +296,11 @@ public class CubeDescManager {
String path = cubeDesc.getResourcePath();
getStore().deleteResource(path);
cubeDescMap.remove(cubeDesc.getName());
- Cuboid.reloadCache(cubeDesc.getName());
}
// remove cubeDesc
public void removeLocalCubeDesc(String name) throws IOException {
cubeDescMap.removeLocal(name);
- Cuboid.reloadCache(name);
}
private void reloadAllCubeDesc() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fad942c..246cbf6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -95,11 +96,26 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonProperty("create_time_utc")
private long createTimeUTC;
+ // cuboid scheduler lazy built
+ transient private CuboidScheduler cuboidScheduler;
+
// default constructor for jackson
public CubeInstance() {
}
- public Segments<CubeSegment> getBuildingSegments() {
+ public CuboidScheduler getCuboidScheduler() {
+ if (cuboidScheduler != null)
+ return cuboidScheduler;
+
+ synchronized (this) {
+ if (cuboidScheduler == null) {
+ cuboidScheduler = getDescriptor().getInitialCuboidScheduler();
+ }
+ }
+ return cuboidScheduler;
+ }
+
+ public List<CubeSegment> getBuildingSegments() {
return segments.getBuildingSegments();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b782a5e..6ebfd99 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -43,6 +43,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.dict.DictionaryInfo;
@@ -306,13 +307,14 @@ public class CubeManager implements IRealizationProvider {
// delete cube instance and cube desc
CubeInstance cube = getCube(cubeName);
- if (deleteDesc && cube.getDescriptor() != null) {
- CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
- }
-
// remove cube and update cache
getStore().deleteResource(cube.getResourcePath());
cubeMap.remove(cube.getName());
+ Cuboid.reloadCache(cube);
+
+ if (deleteDesc && cube.getDescriptor() != null) {
+ CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+ }
// delete cube from project
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
@@ -629,17 +631,20 @@ public class CubeManager implements IRealizationProvider {
* @param cubeName
*/
public CubeInstance reloadCubeLocal(String cubeName) {
- return reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+ CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
+ Cuboid.reloadCache(cubeInstance);
+ return cubeInstance;
}
public void removeCubeLocal(String cubeName) {
CubeInstance cube = cubeMap.get(cubeName);
if (cube != null) {
+ cubeMap.removeLocal(cubeName);
for (CubeSegment segment : cube.getSegments()) {
usedStorageLocation.remove(segment.getUuid());
}
+ Cuboid.reloadCache(cube);
}
- cubeMap.removeLocal(cubeName);
}
public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 358183e..495f9c0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ShardingHash;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
@@ -127,6 +128,10 @@ public class CubeSegment implements IBuildable, ISegment, Serializable {
return getCubeInstance().getDescriptor();
}
+ public CuboidScheduler getCuboidScheduler() {
+ return getCubeInstance().getCuboidScheduler();
+ }
+
public static String makeSegmentName(TSRange tsRange, SegmentRange segRange) {
if (tsRange == null && segRange == null) {
return "FULL_BUILD";
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index acebce4..cd26347 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TblColRef;
public class RowKeySplitter implements java.io.Serializable {
+ private CubeSegment cubeSegment;
private CubeDesc cubeDesc;
private RowKeyColumnIO colIO;
@@ -63,6 +64,7 @@ public class RowKeySplitter implements java.io.Serializable {
}
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
+ this.cubeSegment = cubeSeg;
this.enableSharding = cubeSeg.isEnableSharding();
this.cubeDesc = cubeSeg.getCubeDesc();
IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);
@@ -113,7 +115,7 @@ public class RowKeySplitter implements java.io.Serializable {
offset += RowConstants.ROWKEY_CUBOIDID_LEN;
long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
- Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId);
+ Cuboid cuboid = Cuboid.findById(cubeSegment, lastSplittedCuboidId);
// rowkey columns
for (int i = 0; i < cuboid.getColumns().size(); i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index b71608c..fa83775 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -21,15 +21,15 @@ package org.apache.kylin.cube.cuboid;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
@@ -38,16 +38,15 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
@SuppressWarnings("serial")
public class Cuboid implements Comparable<Cuboid>, Serializable {
- private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
+ //TODO Guava cache may be better
+ private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap();
// smaller is better
public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
@@ -57,10 +56,22 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
}
};
- // this is the only entry point for query to find the right cuboid
- public static Cuboid identifyCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
- long cuboidID = identifyCuboidId(cubeDesc, dimensions, metrics);
- return Cuboid.findById(cubeDesc, cuboidID);
+ // this is the only entry point for query to find the right cuboid for a segment
+ public static Cuboid identifyCuboid(CubeSegment cubeSegment, Set<TblColRef> dimensions,
+ Collection<FunctionDesc> metrics) {
+ return identifyCuboid(cubeSegment.getCuboidScheduler(), dimensions, metrics);
+ }
+
+ // this is the only entry point for query to find the right cuboid for a cube instance
+ public static Cuboid identifyCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensions,
+ Collection<FunctionDesc> metrics) {
+ return identifyCuboid(cubeInstance.getCuboidScheduler(), dimensions, metrics);
+ }
+
+ public static Cuboid identifyCuboid(CuboidScheduler cuboidScheduler, Set<TblColRef> dimensions,
+ Collection<FunctionDesc> metrics) {
+ long cuboidID = identifyCuboidId(cuboidScheduler.getCubeDesc(), dimensions, metrics);
+ return Cuboid.findById(cuboidScheduler, cuboidID);
}
public static long identifyCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -77,140 +88,49 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
return cuboidID;
}
- // for full cube, no need to translate cuboid
- public static Cuboid findForFullCube(CubeDesc cube, long cuboidID) {
+ // for mandatory cuboid, no need to translate cuboid
+ public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) {
return new Cuboid(cube, cuboidID, cuboidID);
}
- public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
- return findById(cube, Bytes.toLong(cuboidID));
+ public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) {
+ return findById(cuboidScheduler, Bytes.toLong(cuboidID));
}
- public static Cuboid findById(CubeDesc cube, long cuboidID) {
- Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cube.getName());
+ public static Cuboid findById(CubeSegment cubeSegment, long cuboidID) {
+ return findById(cubeSegment.getCuboidScheduler(), cuboidID);
+ }
+
+ public static Cuboid findById(CubeInstance cubeInstance, long cuboidID) {
+ return findById(cubeInstance.getCuboidScheduler(), cuboidID);
+ }
+
+ @VisibleForTesting
+ static Cuboid findById(CubeDesc cubeDesc, long cuboidID) {
+ return findById(cubeDesc.getInitialCuboidScheduler(), cuboidID);
+ }
+
+ public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) {
+ Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey());
if (cubeCache == null) {
- cubeCache = new ConcurrentHashMap<Long, Cuboid>();
- CUBOID_CACHE.put(cube.getName(), cubeCache);
+ cubeCache = Maps.newConcurrentMap();
+ CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache);
}
Cuboid cuboid = cubeCache.get(cuboidID);
if (cuboid == null) {
- long validCuboidID = translateToValidCuboid(cube, cuboidID);
- cuboid = new Cuboid(cube, cuboidID, validCuboidID);
+ long validCuboidID = cuboidScheduler.findBestMatchCuboid(cuboidID);
+ cuboid = new Cuboid(cuboidScheduler.getCubeDesc(), cuboidID, validCuboidID);
cubeCache.put(cuboidID, cuboid);
}
return cuboid;
}
- public static boolean isValid(CubeDesc cube, long cuboidID) {
- return cube.getAllCuboids().contains(cuboidID);
- }
-
public static long getBaseCuboidId(CubeDesc cube) {
return cube.getRowkey().getFullMask();
}
public static Cuboid getBaseCuboid(CubeDesc cube) {
- return findById(cube, getBaseCuboidId(cube));
- }
-
- static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
- long baseCuboidId = getBaseCuboidId(cubeDesc);
- if (cubeDesc.getAllCuboids().contains(cuboidID)) {
- return cuboidID;
- }
- List<Long> onTreeCandidates = Lists.newArrayList();
- for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
- Long candidate = translateToOnTreeCuboid(agg, cuboidID);
- if (candidate != null) {
- onTreeCandidates.add(candidate);
- }
- }
-
- if (onTreeCandidates.size() == 0) {
- return baseCuboidId;
- }
-
- long onTreeCandi = Collections.min(onTreeCandidates, cuboidSelectComparator);
- if (isValid(cubeDesc, onTreeCandi)) {
- return onTreeCandi;
- }
-
- return cubeDesc.getCuboidScheduler().findBestMatchCuboid(onTreeCandi);
- }
-
- private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
- if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
- //the partial cube might not contain all required dims
- return null;
- }
-
- // add mandantory
- cuboidID = cuboidID | agg.getMandatoryColumnMask();
-
- // add hierarchy
- for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
- long fullMask = hierarchyMask.fullMask;
- long intersect = cuboidID & fullMask;
- if (intersect != 0 && intersect != fullMask) {
-
- boolean startToFill = false;
- for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
- if (startToFill) {
- cuboidID |= hierarchyMask.dims[i];
- } else {
- if ((cuboidID & hierarchyMask.dims[i]) != 0) {
- startToFill = true;
- cuboidID |= hierarchyMask.dims[i];
- }
- }
- }
- }
- }
-
- // add joint dims
- for (Long joint : agg.getJoints()) {
- if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
- cuboidID = cuboidID | joint;
- }
- }
-
- if (!agg.isOnTree(cuboidID)) {
- // no column, add one column
- long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), agg.getJoints());
- if (nonJointDims != 0) {
- long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
- @Override
- public Long apply(HierarchyMask input) {
- return input.fullMask;
- }
- }));
- if (nonJointNonHierarchy != 0) {
- //there exists dim that does not belong to any joint or any hierarchy, that's perfect
- return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
- } else {
- //choose from a hierarchy that does not intersect with any joint dim, only check level 1
- long allJointDims = agg.getJointDimsMask();
- for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
- long dim = hierarchyMask.allMasks[0];
- if ((dim & allJointDims) == 0) {
- return cuboidID | dim;
- }
- }
- }
- }
-
- cuboidID = cuboidID | Collections.min(agg.getJoints(), cuboidSelectComparator);
- Preconditions.checkState(agg.isOnTree(cuboidID));
- }
- return cuboidID;
- }
-
- private static long removeBits(long original, Collection<Long> toRemove) {
- long ret = original;
- for (Long joint : toRemove) {
- ret = ret & ~joint;
- }
- return ret;
+ return findForMandatory(cube, getBaseCuboidId(cube));
}
// ============================================================================
@@ -310,8 +230,12 @@ public class Cuboid implements Comparable<Cuboid>, Serializable {
CUBOID_CACHE.clear();
}
- public static void reloadCache(String cubeDescName) {
- CUBOID_CACHE.remove(cubeDescName);
+ public static void reloadCache(CubeInstance cubeInstance) {
+ reloadCache(cubeInstance.getCuboidScheduler());
+ }
+
+ private static void reloadCache(CuboidScheduler cuboidScheduler) {
+ CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey());
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index d657a43..4300a6c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -44,7 +44,7 @@ public class CuboidCLI {
}
public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
- CuboidScheduler scheduler = cubeDesc.getCuboidScheduler();
+ CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler();
long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
Collection<Long> cuboidSet = new TreeSet<Long>();
cuboidSet.add(baseCuboid);
@@ -88,28 +88,28 @@ public class CuboidCLI {
//check all valid and invalid
for (long i = 0; i < baseCuboid; ++i) {
if (cuboidSet.contains(i)) {
- if (!Cuboid.isValid(cubeDesc, i)) {
+ if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) {
throw new RuntimeException();
}
- if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
+ if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) != i) {
throw new RuntimeException();
}
} else {
- if (Cuboid.isValid(cubeDesc, i)) {
+ if (cubeDesc.getInitialCuboidScheduler().isValid(i)) {
throw new RuntimeException();
}
- long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
+ long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i);
if (corrected == i) {
throw new RuntimeException();
}
- if (!Cuboid.isValid(cubeDesc, corrected)) {
+ if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) {
throw new RuntimeException();
}
- if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
+ if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected) != corrected) {
throw new RuntimeException();
}
}
@@ -125,7 +125,7 @@ public class CuboidCLI {
long baseCuboid = Cuboid.getBaseCuboidId(cube);
TreeSet<Long> expectedCuboids = new TreeSet<Long>();
for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
- if (Cuboid.isValid(cube, cuboid)) {
+ if (cube.getInitialCuboidScheduler().isValid(cuboid)) {
expectedCuboids.add(cuboid);
}
}
@@ -133,10 +133,10 @@ public class CuboidCLI {
}
public static int[] calculateAllLevelCount(CubeDesc cube) {
- int levels = cube.getBuildLevel();
+ int levels = cube.getInitialBuildLevel();
int[] allLevelCounts = new int[levels + 1];
- CuboidScheduler scheduler = cube.getCuboidScheduler();
+ CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
LinkedList<Long> nextQueue = new LinkedList<Long>();
LinkedList<Long> currentQueue = new LinkedList<Long>();
long baseCuboid = Cuboid.getBaseCuboidId(cube);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index e802230..cef0f77 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Set;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
import com.google.common.base.Preconditions;
@@ -63,14 +62,25 @@ abstract public class CuboidScheduler {
/** Returns a cuboid on the tree that best matches the request cuboid. */
abstract public long findBestMatchCuboid(long requestCuboid);
-
- /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */
- abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg);
+
+ /** Returns whether requestCuboid is valid or not*/
+ abstract public boolean isValid(long requestCuboid);
+
+ /** Returns the key for what this cuboid scheduler responsible for*/
+ abstract public String getResponsibleKey();
// ============================================================================
private transient List<List<Long>> cuboidsByLayer;
+ public long getBaseCuboidId() {
+ return Cuboid.getBaseCuboidId(cubeDesc);
+ }
+
+ public CubeDesc getCubeDesc() {
+ return cubeDesc;
+ }
+
/**
* Get cuboids by layer. It's built from pre-expanding tree.
* @return layered cuboids
@@ -105,4 +115,11 @@ abstract public class CuboidScheduler {
return cuboidsByLayer;
}
+ /**
+ * Get cuboid level count except base cuboid
+ * @return
+ */
+ public int getBuildLevel() {
+ return getCuboidsByLayer().size() - 1;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index f7d22da..cba417a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -35,10 +35,14 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.TooManyCuboidException;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -83,14 +87,48 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
return Sets.newHashSet(allCuboidIds);
}
+ @Override
+ public boolean isValid(long requestCuboid) {
+ return allCuboidIds.contains(requestCuboid);
+ }
+
/**
- * Get the parent cuboid really on the spanning tree.
- * @param child an on-tree cuboid
+ * Get the parent cuboid rely on the spanning tree.
+ * @param cuboid an on-tree cuboid
* @return
*/
@Override
- public long findBestMatchCuboid(long child) {
- long parent = getOnTreeParent(child);
+ public long findBestMatchCuboid(long cuboid) {
+ return findBestMatchCuboid1(cuboid);
+ }
+
+ public long findBestMatchCuboid1(long cuboid) {
+ if (isValid(cuboid)) {
+ return cuboid;
+ }
+
+ List<Long> onTreeCandidates = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ Long candidate = translateToOnTreeCuboid(agg, cuboid);
+ if (candidate != null) {
+ onTreeCandidates.add(candidate);
+ }
+ }
+
+ if (onTreeCandidates.size() == 0) {
+ return getBaseCuboidId();
+ }
+
+ long onTreeCandi = Collections.min(onTreeCandidates, Cuboid.cuboidSelectComparator);
+ if (isValid(onTreeCandi)) {
+ return onTreeCandi;
+ }
+
+ return doFindBestMatchCuboid1(onTreeCandi);
+ }
+
+ public long doFindBestMatchCuboid1(long cuboid) {
+ long parent = getOnTreeParent(cuboid);
while (parent > 0) {
if (cubeDesc.getAllCuboids().contains(parent)) {
break;
@@ -99,11 +137,88 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
}
if (parent <= 0) {
- throw new IllegalStateException("Can't find valid parent for Cuboid " + child);
+ throw new IllegalStateException("Can't find valid parent for Cuboid " + cuboid);
}
return parent;
}
+ private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) {
+ if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+ //the partial cube might not contain all required dims
+ return null;
+ }
+
+ // add mandantory
+ cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+ // add hierarchy
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long fullMask = hierarchyMask.fullMask;
+ long intersect = cuboidID & fullMask;
+ if (intersect != 0 && intersect != fullMask) {
+
+ boolean startToFill = false;
+ for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
+ if (startToFill) {
+ cuboidID |= hierarchyMask.dims[i];
+ } else {
+ if ((cuboidID & hierarchyMask.dims[i]) != 0) {
+ startToFill = true;
+ cuboidID |= hierarchyMask.dims[i];
+ }
+ }
+ }
+ }
+ }
+
+ // add joint dims
+ for (Long joint : agg.getJoints()) {
+ if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+ cuboidID = cuboidID | joint;
+ }
+ }
+
+ if (!agg.isOnTree(cuboidID)) {
+ // no column, add one column
+ long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()),
+ agg.getJoints());
+ if (nonJointDims != 0) {
+ long nonJointNonHierarchy = removeBits(nonJointDims,
+ Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+ @Override
+ public Long apply(HierarchyMask input) {
+ return input.fullMask;
+ }
+ }));
+ if (nonJointNonHierarchy != 0) {
+ //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+ return cuboidID | Long.lowestOneBit(nonJointNonHierarchy);
+ } else {
+ //choose from a hierarchy that does not intersect with any joint dim, only check level 1
+ long allJointDims = agg.getJointDimsMask();
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long dim = hierarchyMask.allMasks[0];
+ if ((dim & allJointDims) == 0) {
+ return cuboidID | dim;
+ }
+ }
+ }
+ }
+
+ cuboidID = cuboidID | Collections.min(agg.getJoints(), Cuboid.cuboidSelectComparator);
+ Preconditions.checkState(agg.isOnTree(cuboidID));
+ }
+ return cuboidID;
+ }
+
+ private static long removeBits(long original, Collection<Long> toRemove) {
+ long ret = original;
+ for (Long joint : toRemove) {
+ ret = ret & ~joint;
+ }
+ return ret;
+ }
+
private long getOnTreeParent(long child) {
Collection<Long> candidates = getOnTreeParents(child);
if (candidates == null || candidates.isEmpty()) {
@@ -259,7 +374,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
* @param agg agg group
* @return cuboidId list
*/
- @Override
public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) {
Set<Long> cuboidHolder = new HashSet<>();
@@ -352,4 +466,40 @@ public class DefaultCuboidScheduler extends CuboidScheduler {
return Long.bitCount(cuboidID) <= dimCap;
}
+ public long findBestMatchCuboid2(long cuboid) {
+ long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc));
+ if (bestParent < -1) {
+ throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid);
+ }
+ return bestParent;
+ }
+
+ private long doFindBestMatchCuboid2(long cuboid, long parent) {
+ if (!canDerive(cuboid, parent)) {
+ return -1;
+ }
+ List<Long> children = parent2child.get(parent);
+ List<Long> candidates = Lists.newArrayList();
+ if (children != null) {
+ for (long child : children) {
+ long candidate = doFindBestMatchCuboid2(cuboid, child);
+ if (candidate > 0) {
+ candidates.add(candidate);
+ }
+ }
+ }
+ if (candidates.isEmpty()) {
+ candidates.add(parent);
+ }
+
+ return Collections.min(candidates, Cuboid.cuboidSelectComparator);
+ }
+
+ private boolean canDerive(long cuboidId, long parentCuboid) {
+ return (cuboidId & ~parentCuboid) == 0;
+ }
+
+ public String getResponsibleKey() {
+ return CubeDesc.class.getName() + "-" + cubeDesc.getName();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
new file mode 100644
index 0000000..414fc4a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.model.CubeDesc;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class TreeCuboidScheduler extends CuboidScheduler {
+
+ final private CuboidTree cuboidTree;
+
+ public TreeCuboidScheduler(CubeDesc cubeDesc, List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+ super(cubeDesc);
+ cuboidTree = CuboidTree.createFromCuboids(allCuboidIds, cuboidComparator);
+ }
+
+ @Override
+ public Set<Long> getAllCuboidIds() {
+ return cuboidTree.getAllCuboidIds();
+ }
+
+ @Override
+ public int getCuboidCount() {
+ return cuboidTree.getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
+ }
+
+ @Override
+ public List<Long> getSpanningCuboid(long cuboidId) {
+ return cuboidTree.getSpanningCuboid(cuboidId);
+ }
+
+ @Override
+ public long findBestMatchCuboid(long cuboidId) {
+ return cuboidTree.findBestMatchCuboid(cuboidId);
+ }
+
+ @Override
+ public boolean isValid(long requestCuboid) {
+ return cuboidTree.isValid(requestCuboid);
+ }
+
+ public static class CuboidTree {
+ private int treeLevels;
+
+ private TreeNode root;
+
+ private Comparator<Long> cuboidComparator;
+
+ private Map<Long, TreeNode> index = new HashMap<>();
+
+ @VisibleForTesting
+ static CuboidTree createFromCuboids(List<Long> allCuboidIds) {
+ return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator);
+ }
+
+ @VisibleForTesting
+ static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+ // sort the cuboid ids in descending order, so that don't need to adjust
+ // the cuboid tree when adding cuboid id to the tree.
+ Collections.sort(allCuboidIds, new Comparator<Long>() {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return Long.compare(o2, o1);
+ }
+ });
+ long basicCuboidId = allCuboidIds.get(0);
+ CuboidTree cuboidTree = new CuboidTree(cuboidComparator);
+ cuboidTree.setRoot(basicCuboidId);
+
+ for (long cuboidId : allCuboidIds) {
+ cuboidTree.addCuboid(cuboidId);
+ }
+ cuboidTree.buildIndex();
+ return cuboidTree;
+ }
+
+ private CuboidTree(Comparator<Long> cuboidComparator) {
+ this.cuboidComparator = cuboidComparator;
+ }
+
+ public Set<Long> getAllCuboidIds() {
+ return index.keySet();
+ }
+
+ public List<Long> getSpanningCuboid(long cuboidId) {
+ TreeNode node = index.get(cuboidId);
+ if (node == null) {
+ throw new IllegalArgumentException("the cuboid:" + cuboidId + " is not exist in the tree");
+ }
+
+ return Lists.transform(node.children, new Function<TreeNode, Long>() {
+ @Nullable
+ @Override
+ public Long apply(@Nullable TreeNode input) {
+ return input.cuboidId;
+ }
+ });
+ }
+
+ public long findBestMatchCuboid(long cuboidId) {
+ // exactly match
+ if (isValid(cuboidId)) {
+ return cuboidId;
+ }
+
+ return findBestParent(cuboidId).cuboidId;
+ }
+
+ public boolean isValid(long cuboidId) {
+ return index.containsKey(cuboidId);
+ }
+
+ private int getCuboidCount(long cuboidId) {
+ int r = 1;
+ for (Long child : getSpanningCuboid(cuboidId)) {
+ r += getCuboidCount(child);
+ }
+ return r;
+ }
+
+ public void print(PrintWriter out) {
+ int dimensionCnt = Long.bitCount(root.cuboidId);
+ doPrint(root, dimensionCnt, 0, out);
+ }
+
+ private void doPrint(TreeNode node, int dimensionCount, int depth, PrintWriter out) {
+ printCuboid(node.cuboidId, dimensionCount, depth, out);
+
+ for (TreeNode child : node.children) {
+ doPrint(child, dimensionCount, depth + 1, out);
+ }
+ }
+
+ private void printCuboid(long cuboidID, int dimensionCount, int depth, PrintWriter out) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < depth; i++) {
+ sb.append(" ");
+ }
+ String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount);
+ sb.append("|---- Cuboid ").append(cuboidName).append("(" + cuboidID + ")");
+ out.println(sb.toString());
+ }
+
+ private void setRoot(long basicCuboidId) {
+ this.root = new TreeNode(basicCuboidId, 0);
+ this.treeLevels = 0;
+ }
+
+ private void buildIndex() {
+ LinkedList<TreeNode> queue = new LinkedList<>();
+ queue.add(root);
+ while (!queue.isEmpty()) {
+ TreeNode node = queue.removeFirst();
+ index.put(node.cuboidId, node);
+ for (TreeNode child : node.children) {
+ queue.add(child);
+ }
+ }
+ }
+
+ private void addCuboid(long cuboidId) {
+ TreeNode parent = findBestParent(cuboidId);
+ if (parent != null && parent.cuboidId != cuboidId) {
+ parent.addChild(cuboidId, parent.level);
+ this.treeLevels = Math.max(this.treeLevels, parent.level + 1);
+ }
+ }
+
+ private TreeNode findBestParent(long cuboidId) {
+ TreeNode bestParent = doFindBestParent(cuboidId, root);
+ if (bestParent == null) {
+ throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboidId);
+ }
+ return bestParent;
+ }
+
+ private TreeNode doFindBestParent(long cuboidId, TreeNode parentCuboid) {
+ if (!canDerive(cuboidId, parentCuboid.cuboidId)) {
+ return null;
+ }
+
+ List<TreeNode> candidates = Lists.newArrayList();
+ for (TreeNode childCuboid : parentCuboid.children) {
+ TreeNode candidate = doFindBestParent(cuboidId, childCuboid);
+ if (candidate != null) {
+ candidates.add(candidate);
+ }
+ }
+ if (candidates.isEmpty()) {
+ candidates.add(parentCuboid);
+ }
+
+ return Collections.min(candidates, new Comparator<TreeNode>() {
+ @Override
+ public int compare(TreeNode o1, TreeNode o2) {
+ return cuboidComparator.compare(o1.cuboidId, o2.cuboidId);
+ }
+ });
+ }
+
+ private boolean canDerive(long cuboidId, long parentCuboid) {
+ return (cuboidId & ~parentCuboid) == 0;
+ }
+ }
+
+ public static class TreeNode {
+ @JsonProperty("cuboid_id")
+ long cuboidId;
+ @JsonIgnore
+ int level;
+ @JsonProperty("children")
+ List<TreeNode> children = new ArrayList<>();
+
+ public long getCuboidId() {
+ return cuboidId;
+ }
+
+ public int getLevel() {
+ return level;
+ }
+
+ public List<TreeNode> getChildren() {
+ return children;
+ }
+
+ TreeNode(long cuboidId, int level) {
+ this.cuboidId = cuboidId;
+ this.level = level;
+ }
+
+ void addChild(long childId, int parentlevel) {
+ this.children.add(new TreeNode(childId, parentlevel + 1));
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32));
+ result = prime * result + level;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ TreeNode other = (TreeNode) obj;
+ if (cuboidId != other.cuboidId)
+ return false;
+ if (level != other.level)
+ return false;
+ return true;
+ }
+ }
+
+ /**
+ * Compare cuboid according to the cuboid data row count
+ */
+ public static class CuboidCostComparator implements Comparator<Long> {
+ private Map<Long, Long> cuboidStatistics;
+
+ public CuboidCostComparator(Map<Long, Long> cuboidStatistics) {
+ Preconditions.checkArgument(cuboidStatistics != null,
+ "the input " + cuboidStatistics + " should not be null!!!");
+ this.cuboidStatistics = cuboidStatistics;
+ }
+
+ @Override
+ public int compare(Long cuboid1, Long cuboid2) {
+ Long rowCnt1 = cuboidStatistics.get(cuboid1);
+ Long rowCnt2 = cuboidStatistics.get(cuboid2);
+ if (rowCnt2 == null || rowCnt1 == null) {
+ return Cuboid.cuboidSelectComparator.compare(cuboid1, cuboid2);
+ }
+ return Long.compare(rowCnt1, rowCnt2);
+ }
+ }
+
+ public String getResponsibleKey() {
+ return CubeInstance.class.getName() + "-" + cubeDesc.getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index c7a4a05..ae38261 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -42,6 +43,7 @@ abstract public class AbstractInMemCubeBuilder {
private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class);
+ final protected CuboidScheduler cuboidScheduler;
final protected IJoinedFlatTableDesc flatDesc;
final protected CubeDesc cubeDesc;
final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
@@ -49,16 +51,23 @@ abstract public class AbstractInMemCubeBuilder {
protected int taskThreadCount = 1;
protected int reserveMemoryMB = 100;
- public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- if (flatDesc == null)
+ // @Deprecated
+ // public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ // this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap);
+ // }
+
+ protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+ Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (cuboidScheduler == null)
throw new NullPointerException();
- if (cubeDesc == null)
+ if (flatDesc == null)
throw new NullPointerException();
if (dictionaryMap == null)
throw new IllegalArgumentException("dictionary cannot be null");
+ this.cuboidScheduler = cuboidScheduler;
this.flatDesc = flatDesc;
- this.cubeDesc = cubeDesc;
+ this.cubeDesc = cuboidScheduler.getCubeDesc();
this.dictionaryMap = dictionaryMap;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index a9211da..dd92a2b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -35,7 +35,7 @@ import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.IGTScanner;
@@ -57,8 +57,9 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
private int splitRowThreshold = Integer.MAX_VALUE;
private int unitRows = 1000;
- public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- super(cubeDesc, flatDesc, dictionaryMap);
+ public DoggedCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+ Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ super(cuboidScheduler, flatDesc, dictionaryMap);
// check memory more often if a single row is big
if (cubeDesc.hasMemoryHungryMeasures())
@@ -276,7 +277,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
RuntimeException exception;
public SplitThread() {
- this.builder = new InMemCubeBuilder(cubeDesc, flatDesc, dictionaryMap);
+ this.builder = new InMemCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap);
this.builder.setConcurrentThreads(taskThreadCount);
this.builder.setReserveMemoryMB(reserveMemoryMB);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 93736c9..684c26b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -39,7 +39,6 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.kv.CubeDimEncMap;
-import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.gridtable.GTAggregateScanner;
import org.apache.kylin.gridtable.GTBuilder;
import org.apache.kylin.gridtable.GTInfo;
@@ -71,7 +70,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private static final double DERIVE_AGGR_CACHE_CONSTANT_FACTOR = 0.1;
private static final double DERIVE_AGGR_CACHE_VARIABLE_FACTOR = 0.9;
- private final CuboidScheduler cuboidScheduler;
private final long baseCuboidId;
private final int totalCuboidCount;
private final String[] metricsAggrFuncs;
@@ -90,9 +88,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private Object[] totalSumForSanityCheck;
private ICuboidCollector resultCollector;
- public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- super(cubeDesc, flatDesc, dictionaryMap);
- this.cuboidScheduler = cubeDesc.getCuboidScheduler();
+ public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
+ Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ super(cuboidScheduler, flatDesc, dictionaryMap);
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
@@ -109,8 +107,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
- GTInfo info = CubeGridTable.newGTInfo(
- Cuboid.findById(cubeDesc, cuboidID),
+ GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cuboidScheduler, cuboidID),
new CubeDimEncMap(cubeDesc, dictionaryMap)
);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index d5948d4..5a1f668 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -36,6 +36,7 @@ import org.apache.kylin.metadata.model.TblColRef;
*/
public class RowKeyDecoder {
+ private final CubeSegment cubeSegment;
private final CubeDesc cubeDesc;
private final RowKeyColumnIO colIO;
private final RowKeySplitter rowKeySplitter;
@@ -44,6 +45,7 @@ public class RowKeyDecoder {
private List<String> values;
public RowKeyDecoder(CubeSegment cubeSegment) {
+ this.cubeSegment = cubeSegment;
this.cubeDesc = cubeSegment.getCubeDesc();
this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
@@ -73,7 +75,7 @@ public class RowKeyDecoder {
if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
return;
}
- this.cuboid = Cuboid.findById(cubeDesc, cuboidID);
+ this.cuboid = Cuboid.findById(cubeSegment, cuboidID);
}
private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index a669fb1..40cda76 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -18,7 +18,11 @@
package org.apache.kylin.cube.kv;
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -28,10 +32,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.base.Preconditions;
public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index af026af..852451d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -28,6 +28,7 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -295,7 +296,8 @@ public class AggregationGroup implements Serializable {
long combination = 1;
if (this.getDimCap() > 0) {
- combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size();
+ DefaultCuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+ combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
} else {
Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3cbba50..dff3485 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -51,7 +51,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
@@ -189,8 +189,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonInclude(JsonInclude.Include.NON_NULL)
private int parentForward = 3;
- transient private CuboidScheduler cuboidScheduler = null;
-
public boolean isEnableSharding() {
//in the future may extend to other storage that is shard-able
return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID;
@@ -459,14 +457,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return true;
}
- /**
- * Get cuboid level count except base cuboid
- * @return
- */
- public int getBuildLevel() {
- return getCuboidScheduler().getCuboidsByLayer().size() - 1;
- }
-
@Override
public int hashCode() {
int result = 0;
@@ -560,7 +550,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
hostToDerivedMap = Maps.newHashMap();
extendedColumnToHosts = Maps.newHashMap();
cuboidBlackSet = Sets.newHashSet();
- cuboidScheduler = null;
}
public void init(KylinConfig config) {
@@ -618,16 +607,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
amendAllColumns();
}
- public CuboidScheduler getCuboidScheduler() {
- if (cuboidScheduler != null)
- return cuboidScheduler;
+ public DefaultCuboidScheduler getInitialCuboidScheduler() {
+ return new DefaultCuboidScheduler(this);
+ }
- synchronized (this) {
- if (cuboidScheduler == null) {
- cuboidScheduler = CuboidScheduler.getInstance(this);
- }
- return cuboidScheduler;
- }
+ /**
+ * Get cuboid level count except base cuboid
+ * @return
+ */
+ public int getInitialBuildLevel() {
+ return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1;
}
public boolean isBlackedCuboid(long cuboidID) {
@@ -1169,7 +1158,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
public Set<Long> getAllCuboids() {
- return getCuboidScheduler().getAllCuboidIds();
+ return getInitialCuboidScheduler().getAllCuboidIds();
}
public int getParentForward() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 223df7c..73da802 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -53,8 +53,7 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, Serializ
// check what columns from hive tables are required, and index them
private void parseCubeDesc() {
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
// build index for rowkey columns
List<TblColRef> cuboidColumns = baseCuboid.getColumns();
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 1c3c395..ec4c04a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -57,7 +57,7 @@ public class CubingUtils {
public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
- final Set<Long> allCuboidIds = cubeDesc.getCuboidScheduler().getAllCuboidIds();
+ final Set<Long> allCuboidIds = cubeDesc.getInitialCuboidScheduler().getAllCuboidIds();
final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 58c0edb..3f3a7c4 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -115,7 +115,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
for (long child : spannings) {
- assertTrue(Cuboid.isValid(cube, child));
+ assertTrue(scheduler.isValid(child));
}
}
@@ -128,7 +128,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testGetSpanningCuboid2() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = cube.getCuboidScheduler();
+ CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
// generate 8d
System.out.println("Spanning for 8D Cuboids");
@@ -156,7 +156,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testGetSpanningCuboid1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = cube.getCuboidScheduler();
+ CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
// generate 7d
System.out.println("Spanning for 7D Cuboids");
@@ -228,7 +228,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -241,7 +241,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts2() {
CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -254,7 +254,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts3() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -267,7 +267,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts4() {
CubeDesc cube = getTestKylinCubeWithSellerLeft();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -280,7 +280,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts5() {
CubeDesc cube = getStreamingCubeDesc();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -293,7 +293,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidCounts6() {
CubeDesc cube = getCIInnerJoinCube();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
int[] counts = CuboidCLI.calculateAllLevelCount(cube);
printCount(counts);
int sum = 0;
@@ -306,7 +306,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testLargeCube() {
CubeDesc cube = getFiftyDimCubeDesc();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
long start = System.currentTimeMillis();
System.out.println(cuboidScheduler.getCuboidCount());
System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
@@ -317,7 +317,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
CubeDesc cube = getTwentyDimCubeDesc();
- CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
cuboidScheduler.getCuboidCount();
twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
}
@@ -332,10 +332,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
CubeDescManager.clearCache();
CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number");
- CuboidScheduler scheduler = cube.getCuboidScheduler();
+ CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
- assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+ assertTrue(scheduler.isValid(baseCuboid.getId()));
List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId());
assertTrue(spanningChild.size() > 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 2e64791..8dc944b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -73,25 +73,26 @@ public class CuboidTest extends LocalFileMetadataTestCase {
public void testIsValid() {
CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
// base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
+ assertEquals(false, cuboidScheduler.isValid(0));
+ assertEquals(true, cuboidScheduler.isValid(toLong("111111111")));
// mandatory column
- assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
- assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("011111110")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("100000000")));
// zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("111111000")));
// aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
- assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("110000111")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("110111000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("111110111")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("111110001")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("111110100")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("110000100")));
}
@Test
@@ -124,41 +125,45 @@ public class CuboidTest extends LocalFileMetadataTestCase {
@Test
public void testIsValid2() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
+
+ assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));
// base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
+ assertEquals(false, cuboidScheduler.isValid(0));
+ assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));
// aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));
// hierarchy
- assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
}
@Test
public void testIsValid3() {
CubeDesc cube = getSSBCubeDesc();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
- assertEquals(false, Cuboid.isValid(cube, toLong("10000000000")));
+ assertEquals(false, cuboidScheduler.isValid(toLong("10000000000")));
// the 4th is mandatory and isMandatoryOnlyValid is true
- assertEquals(true, Cuboid.isValid(cube, toLong("10000001000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("00000001000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("10000001000")));
+ assertEquals(true, cuboidScheduler.isValid(toLong("00000001000")));
}
@Test
public void testFindCuboidByIdWithSingleAggrGroup2() {
CubeDesc cube = getTestKylinCubeWithSeller();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
Cuboid cuboid;
cuboid = Cuboid.findById(cube, 0);
@@ -186,6 +191,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
@Test
public void testFindCuboidByIdWithMultiAggrGroup() {
CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
Cuboid cuboid;
cuboid = Cuboid.findById(cube, toLong("111111110"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 1d1d147..ec1f221 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -91,7 +91,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
data[7] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 75e2458..5af8d8a 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -66,7 +66,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
data[7] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -97,7 +97,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
data[8] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
@@ -133,7 +133,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
data[8] = null;
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index a3af511..a1a39be 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -131,7 +131,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
Set<TblColRef> dimensionsD = new LinkedHashSet<TblColRef>();
dimensionsD.addAll(groupsD);
dimensionsD.addAll(otherDimsD);
- Cuboid cuboid = findCuboid(cubeDesc, dimensionsD, metrics);
+ Cuboid cuboid = findCuboid(cubeInstance, dimensionsD, metrics);
context.setCuboid(cuboid);
// set whether to aggr at storage
@@ -171,8 +171,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
protected abstract String getGTStorage();
- protected Cuboid findCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
- return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
+ protected Cuboid findCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) {
+ return Cuboid.identifyCuboid(cubeInstance, dimensionsD, metrics);
}
protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid,
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index 5db3611..85678ac 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -84,7 +84,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) {
this.cubeSeg = cubeSeg;
long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns);
- this.cuboid = Cuboid.findById(cubeDesc, cuboidId);
+ this.cuboid = Cuboid.findById(cubeSeg, cuboidId);
this.flatOrAndFilter = Lists.newLinkedList();
this.flatOrAndFilter.add(andDimensionRanges);
init(andDimensionRanges);
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 1ec23b6..f64365a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -64,7 +64,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 3: Build Cube
RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
- final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
+ final int groupRowkeyColumnsCount = seg.getCuboidScheduler().getBuildLevel();
// base cuboid step
result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
// n dim cuboid steps
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 106077c..51c9056 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -76,7 +76,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
}
protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
- final int maxLevel = seg.getCubeDesc().getBuildLevel();
+ final int maxLevel = seg.getCuboidScheduler().getBuildLevel();
// base cuboid step
result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
// n dim cuboid steps
http://git-wip-us.apache.org/repos/asf/kylin/blob/12af6533/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
index 07b636b..40f1ac5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -18,7 +18,11 @@
package org.apache.kylin.engine.mr.common;
-import com.google.common.collect.Sets;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeSegment;
@@ -35,10 +39,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.collect.Sets;
/**
*/
@@ -88,8 +89,7 @@ public class BaseCuboidBuilder implements java.io.Serializable {
}
private void init() {
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
initNullBytes();
}