You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/30 04:16:27 UTC
[6/7] incubator-kylin git commit: whitelist
whitelist
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2d5f725b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2d5f725b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2d5f725b
Branch: refs/heads/KYLIN-242
Commit: 2d5f725bf3ea47972ed1fd4c2f67791124abba17
Parents: 2d5d3ef
Author: honma <ho...@ebay.com>
Authored: Fri Nov 27 16:47:28 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 30 11:16:53 2015 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 7 +-
.../persistence/RootPersistentEntity.java | 2 +
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 319 +++----
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 78 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 235 +++--
.../gridtable/CuboidToGridTableMapping.java | 10 +-
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 6 +-
.../org/apache/kylin/cube/kv/RowConstants.java | 2 +
.../kylin/cube/model/AggregationGroup.java | 261 ++++++
.../org/apache/kylin/cube/model/CubeDesc.java | 179 ++--
.../apache/kylin/cube/model/DimensionDesc.java | 121 +--
.../kylin/cube/model/HBaseMappingDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 73 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 227 +----
.../org/apache/kylin/cube/model/SelectRule.java | 30 +
.../kylin/cube/model/v2/CubeBuildTypeEnum.java | 39 +
.../apache/kylin/cube/model/v2/CubeDesc.java | 867 +++++++++++++++++++
.../kylin/cube/model/v2/DimensionDesc.java | 239 +++++
.../kylin/cube/model/v2/HBaseColumnDesc.java | 138 +++
.../cube/model/v2/HBaseColumnFamilyDesc.java | 58 ++
.../kylin/cube/model/v2/HBaseMappingDesc.java | 96 ++
.../kylin/cube/model/v2/HierarchyDesc.java | 68 ++
.../kylin/cube/model/v2/RowKeyColDesc.java | 92 ++
.../apache/kylin/cube/model/v2/RowKeyDesc.java | 295 +++++++
.../model/validation/CubeMetadataValidator.java | 9 +-
.../cube/model/validation/ValidateContext.java | 3 -
.../rule/AggregationGroupOverlapRule.java | 85 ++
.../validation/rule/AggregationGroupRule.java | 170 ++++
.../rule/AggregationGroupSizeRule.java | 66 --
.../rule/IKylinValidationConstants.java | 6 +-
.../validation/rule/MandatoryColumnRule.java | 75 --
.../model/validation/rule/RowKeyAttrRule.java | 16 +-
.../cube/upgrade/V2/CubeDescUpgraderV2.java | 290 +++++++
.../cube/upgrade/V2/CubeMetadataUpgradeV2.java | 177 ++++
.../org/apache/kylin/cube/util/CubingUtils.java | 30 +-
.../kylin/cube/AggregationGroupRuleTest.java | 101 +++
.../cube/AggregationGroupSizeRuleTest.java | 101 ---
.../kylin/cube/DictionaryManagerTest.java | 4 +-
.../kylin/cube/MandatoryColumnRuleTest.java | 57 --
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 64 +-
.../kylin/metadata/MetadataUpgradeTest.java | 2 +-
.../apache/kylin/dict/DictionaryManager.java | 10 +-
.../kylin/metadata/model/MeasureDesc.java | 15 -
.../kylin/engine/mr/BatchCubingJobBuilder.java | 2 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 3 +-
.../mr/steps/MapContextGTRecordWriter.java | 2 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../engine/mr/steps/MergeDictionaryStep.java | 2 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 3 +-
.../apache/kylin/engine/spark/SparkCubing.java | 5 +-
.../spark/cube/DefaultTupleConverter.java | 3 +-
.../localmeta/cube_desc/sample.json | 257 ++++++
.../apache/kylin/rest/service/CubeService.java | 4 +-
.../rest/controller/CubeControllerTest.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 6 +-
.../hbase/cube/v2/CubeTupleConverter.java | 3 +-
.../kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-
.../storage/hbase/steps/CubeHFileMapper.java | 2 +-
.../storage/hbase/steps/CubeHTableUtil.java | 2 +-
.../storage/hbase/steps/HBaseCuboidWriter.java | 2 +-
.../storage/hbase/steps/HBaseMROutput2.java | 4 +-
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../hbase/steps/RowValueDecoderTest.java | 4 +-
68 files changed, 3773 insertions(+), 1274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 5a0fee7..c033117 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -297,10 +297,9 @@ public class FactTableGenerator {
JoinDesc jDesc = dim.getJoin();
if (jDesc == null) {
// column on fact table used directly as a dimension
- for (String aColumn : dim.getColumn()) {
- if (!factTableCol2LookupCol.containsKey(aColumn))
- usedCols.add(aColumn);
- }
+ String aColumn = dim.getColumn();
+ if (!factTableCol2LookupCol.containsKey(aColumn))
+ usedCols.add(aColumn);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index bc72c1e..c8177e6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -90,6 +90,8 @@ abstract public class RootPersistentEntity implements AclEntity, Serializable {
public void setLastModified(long lastModified) {
this.lastModified = lastModified;
}
+
+
public void updateRandomUuid() {
setUuid(UUID.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index d753a20..7c6a57f 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -88,6 +87,7 @@ public class BasicTest {
}
}
+
@Test
public void testxx() throws InterruptedException {
byte[][] data = new byte[10000000][];
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4592b15..4338f54 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -167,7 +167,7 @@ public class CubeManager implements IRealizationProvider {
return null;
DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), "true", col, factTableValueProvider);
+ DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col, factTableValueProvider);
if (dictInfo != null) {
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index d7e7d9c..b0774cf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -19,30 +19,38 @@
package org.apache.kylin.cube.cuboid;
import java.util.ArrayList;
-import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
+
public class Cuboid implements Comparable<Cuboid> {
private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
+ //smaller is better
+ public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return ComparisonChain.start().compare(Long.bitCount(o1), Long.bitCount(o2)).compare(o1, o2).result();
+ }
+ };
+
public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
return findById(cube, Bytes.toLong(cuboidID));
}
@@ -64,29 +72,33 @@ public class Cuboid implements Comparable<Cuboid> {
}
public static boolean isValid(CubeDesc cube, long cuboidID) {
- RowKeyDesc rowkey = cube.getRowkey();
-
- if (cuboidID < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+ if (isValid(agg, cuboidID)) {
+ return true;
+ }
}
- if (checkBaseCuboid(rowkey, cuboidID)) {
- return true;
- }
+ return false;
+ }
- if (checkMandatoryColumns(rowkey, cuboidID) == false) {
- return false;
+ private static boolean isValid(AggregationGroup agg, long cuboidID) {
+ if (cuboidID < 0) {
+ throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
}
-
- if (checkAggregationGroup(rowkey, cuboidID) == false) {
- return false;
+ if ((cuboidID & ~agg.getPartialCubeFullMask()) != 0) {
+ return false; //a cuboid's parent within agg is at most partialCubeFullMask
}
+ return checkMandatoryColumns(agg, cuboidID) && checkHierarchy(agg, cuboidID);
+ }
- if (checkHierarchy(rowkey, cuboidID) == false) {
- return false;
+ public static List<AggregationGroup> getValidAggGroupForCuboid(CubeDesc cubeDesc, long cuboidID) {
+ List<AggregationGroup> ret = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ if (isValid(agg, cuboidID)) {
+ ret.add(agg);
+ }
}
-
- return true;
+ return ret;
}
public static long getBaseCuboidId(CubeDesc cube) {
@@ -97,168 +109,96 @@ public class Cuboid implements Comparable<Cuboid> {
return findById(cube, getBaseCuboidId(cube));
}
- private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
- // add mandantory
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
- if (cuboidID < mandatoryColumnMask) {
- cuboidID = cuboidID | mandatoryColumnMask;
- }
-
- // add hierarchy
- for (DimensionDesc dimension : cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- boolean found = false;
- long result = 0;
- if (hierarchies != null && hierarchies.length > 0) {
- for (int i = hierarchies.length - 1; i >= 0; i--) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = rowkey.getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((rowkey.getTailMask() & bit) > 0)
- continue; // ignore levels in tail, they don't participate
-
- if ((bit & cuboidID) > 0) {
- found = true;
- }
-
- if (found == true) {
- result = result | bit;
- }
- }
- cuboidID = cuboidID | result;
+ public static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
+ List<Long> candidates = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ Long candidate = translateToValidCuboid(agg, cuboidID);
+ if (candidate != null) {
+ candidates.add(candidate);
}
}
- // find the left-most aggregation group
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
-
- if (leftover == 0) {
- return cuboidID;
- }
-
- if (leftover != 0) {
- cuboidID = cuboidID | mask.leftoverMask;
- return cuboidID;
- }
- }
+ if (candidates.size() == 0) {
+ throw new IllegalStateException("Cannot find parent for :" + cuboidID);
}
- // doesn't have column in aggregation groups
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- if (leftover == 0) {
- // doesn't have column in tail group
- if (cuboidWithoutMandatory != 0) {
- return cuboidID;
- } else {
- // no column (except mandatory), add one column
- long toAddCol = (1 << (BitSet.valueOf(new long[] { rowkey.getTailMask() }).cardinality()));
- // check if the toAddCol belongs to any hierarchy
- List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
- if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
- for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
- long result = toAddCol & hierarchyMasks.fullMask;
- if (result > 0) {
- // replace it with the root col in hierarchy
- toAddCol = hierarchyMasks.allMasks[0];
- break;
- }
- }
- }
- cuboidID = cuboidID | toAddCol;
- return cuboidID;
- }
- }
-
- // has column in tail group
- cuboidID = cuboidID | rowkey.getTailMask();
- return cuboidID;
-
+ return Collections.min(candidates, cuboidSelectComparator);
}
- /** Breadth-First-Search
- * @deprecated due to poor performance
- * @param cube
- * @param cuboidID
- * @return
- */
- private static long translateToValidCuboidDeprecated(CubeDesc cube, long cuboidID) {
- if (Cuboid.isValid(cube, cuboidID)) {
- return cuboidID;
+ private static Long translateToValidCuboid(AggregationGroup agg, long cuboidID) {
+ if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+ //the partial cube might not contain all required dims
+ return null;
}
- HashSet<Long> dedupped = new HashSet<Long>();
- Queue<Long> queue = new LinkedList<Long>();
- List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID);
-
- // check each parent
- addToQueue(queue, parents, dedupped);
- while (queue.size() > 0) {
- long parent = pollFromQueue(queue, dedupped);
- if (Cuboid.isValid(cube, parent)) {
- return parent;
- } else {
- addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped);
+ // add mandantory
+ cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+ // add hierarchy
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long fullMask = hierarchyMask.fullMask;
+ long intersect = cuboidID & fullMask;
+ if (intersect != 0 && intersect != fullMask) {
+ long lsb = Long.lowestOneBit(intersect);
+ long fillMask = fullMask & ~(lsb - 1);
+ cuboidID |= fillMask;
}
}
- return -1;
- }
-
- private static List<Long> getAllPossibleParents(CubeDesc cube, long cuboidID) {
- List<Long> allPossibleParents = new ArrayList<Long>();
- for (int i = 0; i < cube.getRowkey().getRowKeyColumns().length; i++) {
- long mask = 1L << i;
- long parentId = cuboidID | mask;
- if (parentId != cuboidID) {
- allPossibleParents.add(parentId);
+ // add joint dims
+ for (Long joint : agg.getJointDims()) {
+ if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+ cuboidID = cuboidID | joint;
}
}
- return allPossibleParents;
- }
-
- private static void addToQueue(Queue<Long> queue, List<Long> parents, HashSet<Long> dedupped) {
- Collections.sort(parents);
- for (Long p : parents) {
- if (!dedupped.contains(p)) {
- dedupped.add(p);
- queue.offer(p);
+ if (cuboidID != 0) {
+ return cuboidID;
+ } else {
+ // no column, add one column
+ long nonJointDims = removeBits(agg.getPartialCubeFullMask(), agg.getJointDims());
+ if (nonJointDims != 0) {
+ long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+ @Override
+ public Long apply(HierarchyMask input) {
+ return input.fullMask;
+ }
+ }));
+ if (nonJointNonHierarchy != 0) {
+ //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+ return Long.lowestOneBit(nonJointNonHierarchy);
+ } else {
+ //choose from a hierarchy that does not intersect with any joint dim, only check level 1
+ long allJointDims = agg.getJointDimsMask();
+ int index = 0;
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long dim = hierarchyMask.allMasks[index];
+ if ((nonJointDims & allJointDims) == 0) {
+ return dim;
+ }
+ }
+ }
}
- }
- }
- private static long pollFromQueue(Queue<Long> queue, HashSet<Long> dedupped) {
- long element = queue.poll();
- dedupped.remove(element);
- return element;
+ return Collections.min(agg.getJointDims(), cuboidSelectComparator);
+ }
}
- private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
- long baseCuboidId = rowkey.getFullMask();
- if (cuboidID > baseCuboidId) {
- throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
+ private static long removeBits(long original, Collection<Long> toRemove) {
+ long ret = original;
+ for (Long joint : toRemove) {
+ ret = ret & ~joint;
}
- return baseCuboidId == cuboidID;
+ return ret;
}
- private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
-
- // note the all-zero cuboid (except for mandatory) is not valid
- if (cuboidID <= mandatoryColumnMask)
- return false;
-
+ private static boolean checkMandatoryColumns(AggregationGroup agg, long cuboidID) {
+ long mandatoryColumnMask = agg.getMandatoryColumnMask();
return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
}
- private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
- List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
+ private static boolean checkHierarchy(AggregationGroup agg, long cuboidID) {
+ List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
// if no hierarchy defined in metadata
if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
return true;
@@ -279,23 +219,9 @@ public class Cuboid implements Comparable<Cuboid> {
return true;
}
- private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
- return leftover == 0 || leftover == mask.leftoverMask;
- }
- }
-
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- return leftover == 0 || leftover == rowkey.getTailMask();
- }
-
// ============================================================================
- private CubeDesc cube;
+ private CubeDesc cubeDesc;
private final long inputID;
private final long id;
private final byte[] idBytes;
@@ -305,8 +231,8 @@ public class Cuboid implements Comparable<Cuboid> {
private volatile CuboidToGridTableMapping cuboidToGridTableMapping = null;
// will translate the cuboidID if it is not valid
- private Cuboid(CubeDesc cube, long originalID, long validID) {
- this.cube = cube;
+ private Cuboid(CubeDesc cubeDesc, long originalID, long validID) {
+ this.cubeDesc = cubeDesc;
this.inputID = originalID;
this.id = validID;
this.idBytes = Bytes.toBytes(id);
@@ -316,7 +242,7 @@ public class Cuboid implements Comparable<Cuboid> {
private List<TblColRef> translateIdToColumns(long cuboidID) {
List<TblColRef> dimesnions = new ArrayList<TblColRef>();
- RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
+ RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
for (int i = 0; i < allColumns.length; i++) {
// NOTE: the order of column in list!!!
long bitmask = 1L << allColumns[i].getBitIndex();
@@ -335,23 +261,30 @@ public class Cuboid implements Comparable<Cuboid> {
// higher level in hierarchy can be ignored when counting aggregation columns
private long eliminateHierarchyAggregation(long id) {
- List<HierarchyMask> hierarchyMaskList = cube.getRowkey().getHierarchyMasks();
- if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
- for (HierarchyMask hierMask : hierarchyMaskList) {
- long[] allMasks = hierMask.allMasks;
- for (int i = allMasks.length - 1; i > 0; i--) {
- long bit = allMasks[i] ^ allMasks[i - 1];
- if ((inputID & bit) != 0) {
- id &= ~allMasks[i - 1];
+ long finalId = id;
+
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ long temp = id;
+ List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
+ if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
+ for (HierarchyMask hierMask : hierarchyMaskList) {
+ long[] allMasks = hierMask.allMasks;
+ for (int i = allMasks.length - 1; i > 0; i--) {
+ long bit = allMasks[i] ^ allMasks[i - 1];
+ if ((inputID & bit) != 0) {
+ temp &= ~allMasks[i - 1];
+ if (temp < finalId)
+ finalId = temp;
+ }
}
}
}
}
- return id;
+ return finalId;
}
- public CubeDesc getCube() {
- return cube;
+ public CubeDesc getCubeDesc() {
+ return cubeDesc;
}
public List<TblColRef> getColumns() {
@@ -375,10 +308,6 @@ public class Cuboid implements Comparable<Cuboid> {
return inputID;
}
- public boolean useAncestor() {
- return inputID != id;
- }
-
public boolean requirePostAggregation() {
return requirePostAggregation;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 7a4b00a..777d809 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -26,9 +26,6 @@ import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
/**
* @author yangli9
@@ -41,11 +38,11 @@ public class CuboidCLI {
if ("test".equals(args[0])) {
CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
- simulateCuboidGeneration(cubeDesc);
+ simulateCuboidGeneration(cubeDesc, true);
}
}
- public static int simulateCuboidGeneration(CubeDesc cube) {
+ public static int simulateCuboidGeneration(CubeDesc cube, boolean validate) {
CuboidScheduler scheduler = new CuboidScheduler(cube);
long baseCuboid = Cuboid.getBaseCuboidId(cube);
@@ -64,20 +61,16 @@ public class CuboidCLI {
cuboidQueue.push(sc);
}
}
-
- /** disable this due to poor performance when dimension number is big
- TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
- if (enumCuboids.equals(cuboidSet) == false) {
- throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
- }
- */
- int mathCount = mathCalcCuboidCount(cube);
- if (mathCount != cuboidSet.size()) {
- throw new IllegalStateException("Math cuboid count " + mathCount + ", but actual cuboid count " + cuboidSet.size());
+ if (validate) {
+ //only run this for test purpose, performance is bad when # of dims is large
+ TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
+ if (enumCuboids.equals(cuboidSet) == false) {
+ throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
+ }
}
- return mathCount;
+ return cuboidSet.size();
}
@@ -93,7 +86,7 @@ public class CuboidCLI {
}
public static int[] calculateAllLevelCount(CubeDesc cube) {
- int levels = cube.getRowkey().getNCuboidBuildLevels();
+ int levels = cube.getBuildLevel();
int[] allLevelCounts = new int[levels + 1];
CuboidScheduler scheduler = new CuboidScheduler(cube);
@@ -116,55 +109,4 @@ public class CuboidCLI {
return allLevelCounts;
}
- public static int mathCalcCuboidCount(CubeDesc cube) {
- int result = 1; // 1 for base cuboid
-
- RowKeyDesc rowkey = cube.getRowkey();
- AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
- result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
- }
-
- return result;
- }
-
- private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
- long groupMask = aggrGroupMask.groupMask;
- int n = mathCalcCuboidCount_combination(rowkey, groupMask);
- n -= 2; // exclude group all 1 and all 0
-
- long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
- if (nonUniqueMask > 0) {
- // exclude duplicates caused by non-unique columns
- // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
- n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
- }
-
- if (hasTail) {
- n *= 2; // tail being 1 and 0
- n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
- }
-
- return n;
- }
-
- private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
- if (colMask == 0) // no column selected
- return 0;
-
- int count = 1;
-
- for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
- long hierBits = colMask & hierMask.fullMask;
- if (hierBits != 0) {
- count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
- colMask &= ~hierBits;
- }
- }
-
- count *= Math.pow(2, Long.bitCount(colMask));
-
- return count;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 7596601..93cab06 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -21,180 +21,171 @@ package org.apache.kylin.cube.cuboid;
/**
*/
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class CuboidScheduler {
- private final CubeDesc cubeDef;
- private final int size;
+ private final CubeDesc cubeDesc;
private final long max;
private final Map<Long, List<Long>> cache;
- public CuboidScheduler(CubeDesc cube) {
- this.cubeDef = cube;
- this.size = cube.getRowkey().getRowKeyColumns().length;
+ public CuboidScheduler(CubeDesc cubeDesc) {
+ this.cubeDesc = cubeDesc;
+ int size = this.cubeDesc.getRowkey().getRowKeyColumns().length;
this.max = (long) Math.pow(2, size) - 1;
this.cache = new ConcurrentHashMap<Long, List<Long>>();
}
- public int getCuboidCount() {
- return getCuboidCount(Cuboid.getBaseCuboidId(cubeDef));
- }
-
- private int getCuboidCount(long cuboidId) {
- int r = 1;
- for (Long child : getSpanningCuboid(cuboidId)) {
- r += getCuboidCount(child);
+ public long getParent(long child) {
+ List<Long> candidates = Lists.newArrayList();
+ long baseCuboidID = Cuboid.getBaseCuboidId(cubeDesc);
+ if (child == baseCuboidID) {
+ throw new IllegalStateException();
}
- return r;
- }
-
- public List<Long> getSpanningCuboid(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
- }
-
- List<Long> result = cache.get(cuboid);
- if (result != null) {
- return result;
- }
-
- // smaller sibling's children
- Collection<Long> allPrevOffspring = new HashSet<Long>();
- for (Long sibling : findSmallerSibling(cuboid)) {
- Collection<Long> prevOffsprings = generateChildren(sibling);
- allPrevOffspring.addAll(prevOffsprings);
- }
-
- // my children is my generation excluding smaller sibling's generation
- result = new ArrayList<Long>();
- for (Long offspring : generateChildren(cuboid)) {
- if (!allPrevOffspring.contains(offspring)) {
- result.add(offspring);
+ for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, child)) {
+ boolean thisAggContributed = false;
+ if (agg.getPartialCubeFullMask() == child) {
+ candidates.add(baseCuboidID);
+ continue;
}
- }
- cache.put(cuboid, result);
- return result;
- }
+ //+1 dim
- private Collection<Long> generateChildren(long cuboid) {
- Collection<Long> result = new HashSet<Long>();
+ //add one normal dim (only try the lowest dim)
+ long normalDimsMask = agg.getNormalDimsMask();
+ if (normalDimsMask != 0) {
+ candidates.add(child | Long.lowestOneBit(normalDimsMask));
+ thisAggContributed = true;
+ }
- // generate zero tail cuboid -- the one with all 1 in the first
- // aggregation group and all 0 for the rest bits
- generateZeroTailBase(cuboid, result);
+ for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+ if ((child & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+ if (i == hierarchyMask.allMasks.length - 1) {
+ continue;//match the full hierarchy
+ }
+ if ((agg.getJointDimsMask() & hierarchyMask.dims[i + 1]) == 0) {
+ //only when the hierarchy dim is not among joints
+ candidates.add(child | hierarchyMask.dims[i + 1]);
+ thisAggContributed = true;
+ }
+ break;//if hierarchyMask 111 is matched, won't check 110 or 100
+ }
+ }
+ }
- RowKeyDesc rowkey = cubeDef.getRowkey();
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if (belongTo(cuboidWithoutMandatory, mask) == false)
+ if (thisAggContributed) {
+ //next section is going to append more than 2 dim to child
+ //thisAggContributed means there's already 1 dim added to child
+ //which can safely prune the 2+ dim candidates.
continue;
+ }
- long[] groupOneBitMasks = mask.groupOneBitMasks;
- for (int i = 0; i < groupOneBitMasks.length; i++) {
- long oneBit = groupOneBitMasks[i];
- if ((cuboid & oneBit) == 0)
- continue;
-
- long child = cuboid ^ oneBit;
- if (Cuboid.isValid(cubeDef, child)) {
- result.add(child);
+ //2+ dim candidates
+ for (long joint : agg.getJointDims()) {
+ if ((child & joint) == 0) {
+ candidates.add(child | joint);
}
}
-
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
}
- return result;
+ return Collections.min(candidates, Cuboid.cuboidSelectComparator);
}
- private void generateZeroTailBase(long cuboid, Collection<Long> result) {
- RowKeyDesc rowkey = cubeDef.getRowkey();
-
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
-
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
- long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
- if (zeroTail > 0 && zeroTail != cuboid) {
- result.add(zeroTail);
+ public Set<Long> getPotentialChildren(long parent) {
+ HashSet<Long> set = Sets.newHashSet();
+ if (parent == Cuboid.getBaseCuboidId(cubeDesc)) {
+ //base cuboid is responsible for spawning each agg group's root
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ long partialCubeFullMask = agg.getPartialCubeFullMask();
+ if (partialCubeFullMask != parent) {
+ set.add(partialCubeFullMask);
}
}
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
+ return set;
}
- }
- public Collection<Long> findSmallerSibling(long cuboid) {
- if (!Cuboid.isValid(cubeDef, cuboid)) {
- return Collections.emptyList();
+ if (Long.bitCount(parent) == 1) {
+ //do not aggregate apex cuboid
+ return set;
}
- RowKeyDesc rowkey = cubeDef.getRowkey();
+ for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, parent)) {
- // do combination in all related groups
- long groupAllBitMask = 0;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((mask.groupMask & cuboid) > 0) {
- groupAllBitMask |= mask.groupMask;
+ //normal dim section
+ for (long normalDimMask : agg.getNormalDims()) {
+ if ((parent & normalDimMask) != 0) {
+ set.add(parent ^ normalDimMask);
+ }
}
+
+ for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+ if ((parent & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+ if ((agg.getJointDimsMask() & hierarchyMask.dims[i]) == 0) {
+ //only when the hierarchy dim is not among joints
+ set.add(parent ^ hierarchyMask.dims[i]);
+ }
+ break;//if hierarchyMask 111 is matched, won't check 110 or 100
+ }
+ }
+ }
+
+ //joint dim section
+ for (long joint : agg.getJointDims()) {
+ if ((parent & joint) == joint) {
+ set.add(parent ^ joint);
+ }
+ }
+
}
- long groupBitValue = cuboid & groupAllBitMask;
- long leftBitValue = cuboid & ~groupAllBitMask;
- long[] groupOneBits = bits(groupAllBitMask);
+ return set;
+ }
- Collection<Long> siblings = new HashSet<Long>();
- combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
- return siblings;
+ public int getCuboidCount() {
+ return getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
}
- private long[] bits(long groupAllBitMask) {
- int size = Long.bitCount(groupAllBitMask);
- long[] r = new long[size];
- long l = groupAllBitMask;
- int i = 0;
- while (l != 0) {
- long bit = Long.highestOneBit(l);
- r[i++] = bit;
- l ^= bit;
+ private int getCuboidCount(long cuboidId) {
+ int r = 1;
+ for (Long child : getSpanningCuboid(cuboidId)) {
+ r += getCuboidCount(child);
}
return r;
}
- private void combination(long cuboid, Collection<Long> siblings, long[] bitMasks, int offset, long bitValue, int k) {
- if (k == 0) {
- if (Cuboid.isValid(cubeDef, bitValue)) {
- siblings.add(bitValue);
- }
- } else {
- for (int i = offset; i < bitMasks.length; i++) {
- long newBitValue = bitValue | bitMasks[i];
- if (newBitValue < cuboid) {
- combination(cuboid, siblings, bitMasks, i + 1, newBitValue, k - 1);
- }
+ public List<Long> getSpanningCuboid(long cuboid) {
+ if (cuboid > max || cuboid < 0) {
+ throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
+ }
+
+ List<Long> result = cache.get(cuboid);
+ if (result != null) {
+ return result;
+ }
+
+ result = Lists.newArrayList();
+ Set<Long> potentials = getPotentialChildren(cuboid);
+ for (Long potential : potentials) {
+ if (getParent(potential) == cuboid) {
+ result.add(potential);
}
}
- }
- private boolean belongTo(long cuboidWithoutMandatory, AggrGroupMask mask) {
- long groupBits = cuboidWithoutMandatory & mask.groupMask;
- long leftoverBits = cuboidWithoutMandatory & mask.leftoverMask;
- return groupBits > 0 && (leftoverBits == 0 || leftoverBits == mask.leftoverMask);
+ cache.put(cuboid, result);
+ return result;
}
public int getCardinality(long cuboid) {
@@ -206,7 +197,7 @@ public class CuboidScheduler {
}
public List<Long> getAllCuboidIds() {
- final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDef);
+ final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
List<Long> result = Lists.newArrayList();
getSubCuboidIds(baseCuboidId, result);
return result;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index c95e932..428137a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -61,7 +61,7 @@ public class CuboidToGridTableMapping {
// column blocks of metrics
ArrayList<BitSet> metricsColBlocks = Lists.newArrayList();
- for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (int i = 0; i < familyDesc.getColumns().length; i++) {
metricsColBlocks.add(new BitSet());
}
@@ -69,7 +69,7 @@ public class CuboidToGridTableMapping {
// metrics
metrics2gt = LinkedListMultimap.create();
- for (MeasureDesc measure :cuboid.getCube().getMeasures()) {
+ for (MeasureDesc measure :cuboid.getCubeDesc().getMeasures()) {
// Count distinct & holistic count distinct are equals() but different.
// Ensure the holistic version if exists is always the first.
FunctionDesc func = measure.getFunction();
@@ -85,7 +85,7 @@ public class CuboidToGridTableMapping {
// map to column block
int cbIdx = 0;
- for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
if (hbaseColDesc.containsMeasure(measure.getName())) {
metricsColBlocks.get(cbIdx).set(gtColIdx);
@@ -102,7 +102,7 @@ public class CuboidToGridTableMapping {
}
nMetrics = gtColIdx - nDimensions;
- assert nMetrics == cuboid.getCube().getMeasures().size();
+ assert nMetrics == cuboid.getCubeDesc().getMeasures().size();
}
public int getColumnCount() {
@@ -156,7 +156,7 @@ public class CuboidToGridTableMapping {
public Map<Integer, Integer> getDependentMetricsMap() {
Map<Integer, Integer> result = Maps.newHashMap();
- List<MeasureDesc> measures = cuboid.getCube().getMeasures();
+ List<MeasureDesc> measures = cuboid.getCubeDesc().getMeasures();
for (MeasureDesc child : measures) {
if (child.getDependentMeasureRef() != null) {
boolean ok = false;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index f0ee372..2fbd45a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -54,11 +54,11 @@ public final class InMemCubeBuilderUtils {
}
public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
+ int cardinality = Long.bitCount(cuboidId);
BitSet dimension = new BitSet();
- dimension.set(0, bitSet.cardinality());
+ dimension.set(0, cardinality);
BitSet metrics = new BitSet();
- metrics.set(bitSet.cardinality(), bitSet.cardinality() + measureCount);
+ metrics.set(cardinality, cardinality + measureCount);
return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
index 62dea02..3510915 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.kv;
public class RowConstants {
+ public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
+
// row key fixed length place holder
public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
// row key lower bound
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
new file mode 100644
index 0000000..2e57583
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class AggregationGroup {
+ public static class HierarchyMask {
+ public long fullMask;// 00000111
+ public long[] allMasks;// 00000100,00000110,00000111
+ public long[] dims;// 00000100,00000100,00000001
+ }
+
+ @JsonProperty("includes")
+ private String[] includes;
+ @JsonProperty("select_rule")
+ private SelectRule selectRule;
+
+ //computed
+ private long partialCubeFullMask;
+ private long mandatoryColumnMask;
+ private List<HierarchyMask> hierarchyMasks;
+ private List<Long> jointDims;//each long is a group
+ private long jointDimsMask;
+ private long normalDimsMask;
+ private long hierarchyDimsMask;
+ private List<Long> normalDims;//each long is a single dim
+
+ public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) {
+ Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation();
+
+ buildPartialCubeFullMask(colNameAbbr, rowKeyDesc);
+ buildMandatoryColumnMask(colNameAbbr, rowKeyDesc);
+ buildHierarchyMasks(colNameAbbr, rowKeyDesc);
+ buildJointColumnMask(colNameAbbr, rowKeyDesc);
+ buildJointDimsMask();
+ buildNormalDimsMask();
+ buildHierarchyDimsMask();
+ }
+
+ private void buildPartialCubeFullMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ Preconditions.checkState(this.includes != null);
+ Preconditions.checkState(this.includes.length != 0);
+
+ partialCubeFullMask = 0L;
+ for (String dim : this.includes) {
+ TblColRef hColumn = colNameAbbr.get(dim);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+ partialCubeFullMask |= bit;
+ }
+ }
+
+ private void buildJointColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ jointDims = Lists.newArrayList();
+
+ for (String[] joint_dims : this.selectRule.joint_dims) {
+ if (joint_dims == null || joint_dims.length == 0) {
+ continue;
+ }
+
+ long joint = 0L;
+ for (int i = 0; i < joint_dims.length; i++) {
+ TblColRef hColumn = colNameAbbr.get(joint_dims[i]);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+ joint |= bit;
+ }
+
+ Preconditions.checkState(joint != 0);
+ jointDims.add(joint);
+ }
+ }
+
+ private void buildMandatoryColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ mandatoryColumnMask = 0L;
+
+ String[] mandatory_dims = this.selectRule.mandatory_dims;
+ if (mandatory_dims == null || mandatory_dims.length == 0) {
+ return;
+ }
+
+ for (String dim : mandatory_dims) {
+ TblColRef hColumn = colNameAbbr.get(dim);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ mandatoryColumnMask |= 1 << index;
+ }
+
+ }
+
+ private void buildHierarchyMasks(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+
+ HierarchyMask mask = new HierarchyMask();
+ this.hierarchyMasks = new ArrayList<HierarchyMask>();
+
+ for (String[] hierarchy_dims : this.selectRule.hierarchy_dims) {
+ if (hierarchy_dims == null || hierarchy_dims.length == 0) {
+ continue;
+ }
+
+ ArrayList<Long> allMaskList = new ArrayList<Long>();
+ ArrayList<Long> dimList = new ArrayList<Long>();
+ for (int i = 0; i < hierarchy_dims.length; i++) {
+ TblColRef hColumn = colNameAbbr.get(hierarchy_dims[i]);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+
+ // if ((tailMask & bit) > 0)
+ // continue; // ignore levels in tail, they don't participate
+ // // aggregation group combination anyway
+
+ mask.fullMask |= bit;
+ allMaskList.add(mask.fullMask);
+ dimList.add(bit);
+ }
+
+ Preconditions.checkState(allMaskList.size() == dimList.size());
+ mask.allMasks = new long[allMaskList.size()];
+ mask.dims = new long[dimList.size()];
+ for (int i = 0; i < allMaskList.size(); i++) {
+ mask.allMasks[i] = allMaskList.get(i);
+ mask.dims[i] = dimList.get(i);
+ }
+
+ this.hierarchyMasks.add(mask);
+
+ }
+
+ }
+
+ private void buildNormalDimsMask() {
+ //no joint, no hierarchy, no mandatory
+ long leftover = partialCubeFullMask & ~mandatoryColumnMask;
+ leftover &= ~this.jointDimsMask;
+ for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+ leftover &= ~hierarchyMask.fullMask;
+ }
+
+ this.normalDimsMask = leftover;
+ this.normalDims = bits(leftover);
+ }
+
+ private void buildHierarchyDimsMask() {
+ long ret = 0;
+ for (HierarchyMask mask : hierarchyMasks) {
+ ret |= mask.fullMask;
+ }
+ this.hierarchyDimsMask = ret;
+ }
+
+ private List<Long> bits(long x) {
+ List<Long> r = Lists.newArrayList();
+ long l = x;
+ while (l != 0) {
+ long bit = Long.lowestOneBit(l);
+ r.add(bit);
+ l ^= bit;
+ }
+ return r;
+ }
+
+ public void buildJointDimsMask() {
+ long ret = 0;
+ for (long x : jointDims) {
+ ret |= x;
+ }
+ this.jointDimsMask = ret;
+ }
+
+ public long getMandatoryColumnMask() {
+ return mandatoryColumnMask;
+ }
+
+ public List<HierarchyMask> getHierarchyMasks() {
+ return hierarchyMasks;
+ }
+
+ public int getBuildLevel() {
+ int ret = 0;
+ ret += getNormalDims().size();
+ long allHierarchyMask = 0L;
+ for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+ ret += hierarchyMask.allMasks.length;
+ allHierarchyMask |= hierarchyMask.fullMask;
+ }
+ for (Long joint : jointDims) {
+ if ((joint & allHierarchyMask) == 0) {
+ ret += 1;
+ }
+ }
+ return ret - 1;
+ }
+
+ public void setIncludes(String[] includes) {
+ this.includes = includes;
+ }
+
+ public void setSelectRule(SelectRule selectRule) {
+ this.selectRule = selectRule;
+ }
+
+ public List<Long> getJointDims() {
+ return jointDims;
+ }
+
+ public long getJointDimsMask() {
+ return jointDimsMask;
+ }
+
+ public long getNormalDimsMask() {
+ return normalDimsMask;
+ }
+
+ public long getHierarchyDimsMask() {
+ return hierarchyDimsMask;
+ }
+
+ public List<Long> getNormalDims() {
+ return normalDims;
+ }
+
+ public long getPartialCubeFullMask() {
+ return partialCubeFullMask;
+ }
+
+ public String[] getIncludes() {
+ return includes;
+ }
+
+ public SelectRule getSelectRule() {
+ return selectRule;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2250945..2df9635 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -32,6 +31,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -58,6 +59,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -67,7 +70,7 @@ import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeDesc extends RootPersistentEntity {
- public static enum DeriveType {
+ public enum DeriveType {
LOOKUP, PK_FK
}
@@ -110,6 +113,8 @@ public class CubeDesc extends RootPersistentEntity {
private RowKeyDesc rowkey;
@JsonProperty("hbase_mapping")
private HBaseMappingDesc hbaseMapping;
+ @JsonProperty("aggregation_gropus")
+ private List<AggregationGroup> aggregationGroups;
@JsonProperty("signature")
private String signature;
@JsonProperty("notify_list")
@@ -171,16 +176,14 @@ public class CubeDesc extends RootPersistentEntity {
}
/**
- * Find FunctionDesc by Full Expression.
- *
- * @return
+ * @return all functions from each measure.
*/
- public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
+ public List<FunctionDesc> listAllFunctions() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
for (MeasureDesc m : measures) {
- if (m.getFunction().equals(manualFunc))
- return m.getFunction();
+ functions.add(m.getFunction());
}
- return null;
+ return functions;
}
public TblColRef findColumnRef(String table, String column) {
@@ -191,14 +194,6 @@ public class CubeDesc extends RootPersistentEntity {
return cols.get(column);
}
- public DimensionDesc findDimensionByColumn(TblColRef col) {
- for (DimensionDesc dim : dimensions) {
- if (ArrayUtils.contains(dim.getColumnRefs(), col))
- return dim;
- }
- return null;
- }
-
public DimensionDesc findDimensionByTable(String lookupTableName) {
lookupTableName = lookupTableName.toUpperCase();
for (DimensionDesc dim : dimensions)
@@ -207,28 +202,6 @@ public class CubeDesc extends RootPersistentEntity {
return null;
}
- public DimensionDesc findDimensionByName(String dimName) {
- dimName = dimName.toUpperCase();
- for (DimensionDesc dim : dimensions) {
- if (dimName.equals(dim.getName()))
- return dim;
- }
- return null;
- }
-
- /**
- * Get all functions from each measure.
- *
- * @return
- */
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measures) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
public boolean isDerived(TblColRef col) {
return derivedToHostMap.containsKey(col);
}
@@ -270,14 +243,6 @@ public class CubeDesc extends RootPersistentEntity {
// ============================================================================
- public HBaseMappingDesc getHBaseMapping() {
- return hbaseMapping;
- }
-
- public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
public KylinConfig getConfig() {
return config;
}
@@ -354,6 +319,14 @@ public class CubeDesc extends RootPersistentEntity {
this.rowkey = rowkey;
}
+ public List<AggregationGroup> getAggregationGroups() {
+ return aggregationGroups;
+ }
+
+ public void setAggregationGroups(List<AggregationGroup> aggregationGroups) {
+ this.aggregationGroups = aggregationGroups;
+ }
+
public String getSignature() {
return signature;
}
@@ -389,12 +362,23 @@ public class CubeDesc extends RootPersistentEntity {
if (!name.equals(cubeDesc.name))
return false;
+
if (!getFactTable().equals(cubeDesc.getFactTable()))
return false;
return true;
}
+ public int getBuildLevel() {
+ return Collections.max(Collections2.transform(aggregationGroups, new Function<AggregationGroup, Integer>() {
+ @Nullable
+ @Override
+ public Integer apply(AggregationGroup input) {
+ return input.getBuildLevel();
+ }
+ })) + 1;
+ }
+
@Override
public int hashCode() {
int result = 0;
@@ -414,13 +398,19 @@ public class CubeDesc extends RootPersistentEntity {
}
return calculateSignature().equals(getSignature());
}
-
+
public String calculateSignature() {
- MessageDigest md = null;
+ MessageDigest md;
try {
md = MessageDigest.getInstance("MD5");
StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
+ sigString.append(this.name).append("|")//
+ .append(JsonUtil.writeValueAsString(this.modelName)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.dimensions)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.measures)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.rowkey)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.aggregationGroups)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.hbaseMapping));
byte[] signature = md.digest(sigString.toString().getBytes());
return new String(Base64.encodeBase64(signature));
@@ -455,11 +445,14 @@ public class CubeDesc extends RootPersistentEntity {
dim.init(this, tables);
}
- sortDimAndMeasure();
initDimensionColumns();
initMeasureColumns();
rowkey.init(this);
+ for (AggregationGroup agg : this.aggregationGroups) {
+ agg.init(this, rowkey);
+ }
+
if (hbaseMapping != null) {
hbaseMapping.init(this);
}
@@ -479,35 +472,32 @@ public class CubeDesc extends RootPersistentEntity {
// init dimension columns
ArrayList<TblColRef> dimCols = Lists.newArrayList();
- String[] colStrs = dim.getColumn();
+ String colStrs = dim.getColumn();
// when column is omitted, special case
- if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
+ if ((colStrs == null && dim.isDerived()) || ("{FK}".equalsIgnoreCase(colStrs))) {
for (TblColRef col : join.getForeignKeyColumns()) {
dimCols.add(initDimensionColRef(col));
}
}
// normal case
else {
- if (colStrs == null || colStrs.length == 0)
+ if (StringUtils.isEmpty(colStrs))
throw new IllegalStateException("Dimension column must not be blank " + dim);
- for (String colStr : colStrs) {
- dimCols.add(initDimensionColRef(dim, colStr));
- }
+ dimCols.add(initDimensionColRef(dim, colStrs));
- // fill back column ref in hierarchy
- if (dim.isHierarchy()) {
- for (int i = 0; i < dimCols.size(); i++)
- dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
- }
+ // // fill back column ref in hierarchy
+ // if (dim.isHierarchy()) {
+ // for (int i = 0; i < dimCols.size(); i++)
+ // dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
+ // }
}
- TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
+ TblColRef[] dimColArray = dimCols.toArray(new TblColRef[dimCols.size()]);
dim.setColumnRefs(dimColArray);
// init derived columns
- TblColRef[] hostCols = dimColArray;
if (dim.isDerived()) {
String[] derived = dim.getDerived();
String[][] split = splitDerivedColumnAndExtra(derived);
@@ -517,7 +507,7 @@ public class CubeDesc extends RootPersistentEntity {
for (int i = 0; i < derivedNames.length; i++) {
derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
}
- initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+ initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
}
// PK-FK derive the other side
@@ -528,10 +518,10 @@ public class CubeDesc extends RootPersistentEntity {
allColumns.addAll(Arrays.asList(fk));
allColumns.addAll(Arrays.asList(pk));
for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, fk[i]);
+ int find = ArrayUtils.indexOf(dimColArray, fk[i]);
if (find >= 0) {
TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
+ initDerivedMap(dimColArray[find], DeriveType.PK_FK, dim, derivedCol);
}
}
/** disable this code as we don't need fk be derived from pk
@@ -693,7 +683,7 @@ public class CubeDesc extends RootPersistentEntity {
for (int i = 0; i < measures.size(); i++)
measureIndexLookup.put(measures.get(i).getName(), i);
- for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc c : cf.getColumns()) {
String[] colMeasureRefs = c.getMeasureRefs();
MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length];
@@ -709,53 +699,6 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- private void sortDimAndMeasure() {
- sortDimensionsByID();
- sortMeasuresByID();
- for (DimensionDesc dim : dimensions) {
- sortHierarchiesByLevel(dim.getHierarchy());
- }
- }
-
- private void sortDimensionsByID() {
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc d1, DimensionDesc d2) {
- Integer id1 = d1.getId();
- Integer id2 = d2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortMeasuresByID() {
- if (measures == null) {
- measures = Lists.newArrayList();
- }
-
- Collections.sort(measures, new Comparator<MeasureDesc>() {
- @Override
- public int compare(MeasureDesc m1, MeasureDesc m2) {
- Integer id1 = m1.getId();
- Integer id2 = m2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
- if (hierarchies != null) {
- Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
- @Override
- public int compare(HierarchyDesc h1, HierarchyDesc h2) {
- Integer level1 = Integer.parseInt(h1.getLevel());
- Integer level2 = Integer.parseInt(h2.getLevel());
- return level1.compareTo(level2);
- }
- });
- }
- }
-
public boolean hasHolisticCountDistinctMeasures() {
for (MeasureDesc measure : measures) {
if (measure.getFunction().isHolisticCountDistinct()) {
@@ -822,7 +765,7 @@ public class CubeDesc extends RootPersistentEntity {
return storageType;
}
- void setStorageType(int storageType) {
+ public void setStorageType(int storageType) {
this.storageType = storageType;
}
@@ -830,7 +773,7 @@ public class CubeDesc extends RootPersistentEntity {
return engineType;
}
- void setEngineType(int engineType) {
+ public void setEngineType(int engineType) {
this.engineType = engineType;
}
@@ -855,7 +798,6 @@ public class CubeDesc extends RootPersistentEntity {
return measureDisplayColumns;
}
-
public boolean hasMeasureUsingDictionary() {
for (MeasureDesc measureDesc : this.getMeasures()) {
if (measureDesc.getFunction().isTopN())
@@ -864,4 +806,5 @@ public class CubeDesc extends RootPersistentEntity {
return false;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index e75a123..bccae58 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -18,11 +18,10 @@
package org.apache.kylin.cube.model;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
@@ -32,34 +31,27 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class DimensionDesc {
- @JsonProperty("id")
- private int id;
-
@JsonProperty("name")
private String name;
-
- @JsonProperty("hierarchy")
- private boolean isHierarchy;
@JsonProperty("table")
private String table;
@JsonProperty("column")
- private String[] column;
+ private String column;
@JsonProperty("derived")
private String[] derived;
private TableDesc tableDesc;
private JoinDesc join;
- private HierarchyDesc[] hierarchy;
// computed
private TblColRef[] columnRefs;
- private TblColRef[] derivedColRefs;
public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
if (name != null)
@@ -80,29 +72,30 @@ public class DimensionDesc {
}
}
- if (isHierarchy && this.column.length > 0) {
- List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
- for (int i = 0, n = this.column.length; i < n; i++) {
- String aColumn = this.column[i];
- HierarchyDesc aHierarchy = new HierarchyDesc();
- aHierarchy.setLevel(String.valueOf(i + 1));
- aHierarchy.setColumn(aColumn);
- hierarchyList.add(aHierarchy);
- }
-
- this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
- }
+ // if (isHierarchy && this.column.length > 0) {
+ // List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
+ // for (int i = 0, n = this.column.length; i < n; i++) {
+ // String aColumn = this.column[i];
+ // HierarchyDesc aHierarchy = new HierarchyDesc();
+ // aHierarchy.setLevel(String.valueOf(i + 1));
+ // aHierarchy.setColumn(aColumn);
+ // hierarchyList.add(aHierarchy);
+ // }
+ //
+ // this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
+ // }
+ //
+ // if (hierarchy != null && hierarchy.length == 0)
+ // hierarchy = null;
+
+ // if (hierarchy != null) {
+ // for (HierarchyDesc h : hierarchy)
+ // h.setColumn(h.getColumn().toUpperCase());
+ // }
- if (hierarchy != null && hierarchy.length == 0)
- hierarchy = null;
if (derived != null && derived.length == 0)
derived = null;
- if (hierarchy != null) {
- for (HierarchyDesc h : hierarchy)
- h.setColumn(h.getColumn().toUpperCase());
- }
-
if (derived != null) {
StringUtil.toUpperCaseArray(derived, derived);
}
@@ -112,29 +105,10 @@ public class DimensionDesc {
}
}
- public boolean isHierarchyColumn(TblColRef col) {
- if (hierarchy == null)
- return false;
-
- for (HierarchyDesc hier : hierarchy) {
- if (hier.getColumnRef().equals(col))
- return true;
- }
- return false;
- }
-
public boolean isDerived() {
return derived != null;
}
- public boolean isHierarchy() {
- return isHierarchy;
- }
-
- public void setHierarchy(boolean isHierarchy) {
- this.isHierarchy = isHierarchy;
- }
-
public String getTable() {
return table;
}
@@ -143,14 +117,6 @@ public class DimensionDesc {
this.table = table;
}
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
public JoinDesc getJoin() {
return join;
}
@@ -171,22 +137,14 @@ public class DimensionDesc {
this.columnRefs = colRefs;
}
- public String[] getColumn() {
+ public String getColumn() {
return this.column;
}
- public void setColumn(String[] column) {
+ public void setColumn(String column) {
this.column = column;
}
- public HierarchyDesc[] getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(HierarchyDesc[] hierarchy) {
- this.hierarchy = hierarchy;
- }
-
public String[] getDerived() {
return derived;
}
@@ -195,45 +153,22 @@ public class DimensionDesc {
this.derived = derived;
}
- public TblColRef[] getDerivedColRefs() {
- return derivedColRefs;
- }
-
- public void setDerivedColRefs(TblColRef[] derivedColRefs) {
- this.derivedColRefs = derivedColRefs;
- }
-
public TableDesc getTableDesc() {
return this.tableDesc;
}
@Override
public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- DimensionDesc that = (DimensionDesc) o;
-
- if (id != that.id)
- return false;
- if (!name.equals(that.name))
- return false;
-
- return true;
+ throw new NotImplementedException();
}
@Override
public int hashCode() {
- int result = id;
- result = 31 * result + name.hashCode();
- return result;
+ throw new NotImplementedException();
}
@Override
public String toString() {
- return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
+ return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index e390bd6..2ef1e17 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -43,7 +43,7 @@ public class HBaseMappingDesc {
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
- HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
+ HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping();
if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 278b59f..7b12cd7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -18,11 +18,15 @@
package org.apache.kylin.cube.model;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
/**
* @author yangli9
@@ -31,37 +35,74 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class RowKeyColDesc {
+ public enum ColEncodingType {
+ DICT, FIXED_LEN
+ }
+
+ public class ColEncoding {
+ public ColEncodingType type;
+ public Object param;
+
+ public ColEncoding(ColEncodingType type, Object param) {
+ this.type = type;
+ this.param = param;
+ }
+ }
+
@JsonProperty("column")
private String column;
- @JsonProperty("length")
- private int length;
- @JsonProperty("dictionary")
- private String dictionary;
- @JsonProperty("mandatory")
- private boolean mandatory = false;
+ @JsonProperty("encoding")
+ private String encoding;
// computed
+ private ColEncoding colEncoding;
private int bitIndex;
private TblColRef colRef;
- public String getDictionary() {
- return dictionary;
+ public void init() {
+
+ //dict or fix length?
+ Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
+ if (this.encoding.equalsIgnoreCase("dict")) {
+ this.colEncoding = new ColEncoding(ColEncodingType.DICT, null);
+ } else if (this.encoding.startsWith("fixed_length")) {
+ int length = RowConstants.ROWKEY_COL_DEFAULT_LENGTH;
+ if (this.encoding.indexOf(":") > 0) {
+ length = Integer.parseInt(this.encoding.substring(this.encoding.indexOf(":") + 1));
+ }
+ this.colEncoding = new ColEncoding(ColEncodingType.FIXED_LEN, length);
+ } else {
+ throw new IllegalArgumentException("Not supported row key col encoding:" + this.encoding);
+ }
+ }
+
+ public String getEncoding() {
+ return encoding;
+ }
+
+ public void setEncoding(String encoding) {
+ this.encoding = encoding;
}
public String getColumn() {
return column;
}
- void setColumn(String column) {
+ public void setColumn(String column) {
this.column = column;
}
- public int getLength() {
- return length;
+ public boolean isUsingDictionary() {
+ return this.colEncoding.type == ColEncodingType.DICT;
+
}
- public boolean isMandatory() {
- return mandatory;
+ public int getLength() {
+ if (this.colEncoding.type == ColEncodingType.FIXED_LEN) {
+ return (Integer) this.colEncoding.param;
+ } else {
+ throw new IllegalStateException("Not knowing the col's length");
+ }
}
public int getBitIndex() {
@@ -80,13 +121,9 @@ public class RowKeyColDesc {
this.colRef = colRef;
}
- public void setDictionary(String dictionary) {
- this.dictionary = dictionary;
- }
-
@Override
public String toString() {
- return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
+ return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
}
}