You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/30 04:16:22 UTC
[1/7] incubator-kylin git commit: more checks on
find-hive-dependency.sh
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-242 [created] 2dda19092
more checks on find-hive-dependency.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a63242cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a63242cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a63242cf
Branch: refs/heads/KYLIN-242
Commit: a63242cf90f84315e2f533279ebbfb72ad812b30
Parents: ccf0207
Author: honma <ho...@ebay.com>
Authored: Mon Nov 23 13:25:05 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 23 13:25:05 2015 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a63242cf/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 4982c93..de14260 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -23,9 +23,14 @@ done
# in some versions of hdp hcatalog is not in hive's classpath, find it separately
if [ -z "$HCAT_HOME" ]
then
- echo "HCAT_HOME not found, try to find hcatalog path from hdp home"
- hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
- hcatalog_home=${hdp_home}/hive-hcatalog
+ echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
+ hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+ if [ -d "${hadoop_home}/hive-hcatalog" ]; then
+ hcatalog_home=${hadoop_home}/hive-hcatalog
+ else
+ echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
+ exit 1
+ fi
else
echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
hcatalog_home=${HCAT_HOME}
[4/7] incubator-kylin git commit: whitelist
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
new file mode 100644
index 0000000..4e7b99e
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ * find forbid overlaps in each AggregationGroup
+ * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint
+ */
+public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
+ * , org.apache.kylin.metadata.validation.ValidateContext)
+ */
+ @Override
+ public void validate(CubeDesc cube, ValidateContext context) {
+ innerValidateMaxSize(cube, context);
+ }
+
+ /**
+ * @param cube
+ * @param context
+ */
+ private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
+ int maxSize = getMaxAgrGroupSize();
+
+ int index = 0;
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+ if (agg.getIncludes() == null) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " include dims not set");
+ continue;
+ }
+
+ if (agg.getSelectRule() == null) {
+ continue;
+ }
+
+ Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getIncludes() != null) {
+ for (String include : agg.getIncludes()) {
+ includeDims.add(include);
+ }
+ }
+
+ Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().mandatory_dims != null) {
+ for (String include : agg.getSelectRule().mandatory_dims) {
+ mandatoryDims.add(include);
+ }
+ }
+
+ Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().hierarchy_dims != null) {
+ for (String[] ss : agg.getSelectRule().hierarchy_dims) {
+ for (String s : ss)
+ hierarchyDims.add(s);
+ }
+ }
+
+ Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().joint_dims != null) {
+ for (String[] ss : agg.getSelectRule().joint_dims) {
+ for (String s : ss)
+ jointDims.add(s);
+ }
+ }
+
+ if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " Include dims not containing all the used dims");
+ }
+
+ Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ normalDims.addAll(includeDims);
+ normalDims.removeAll(mandatoryDims);
+ normalDims.removeAll(hierarchyDims);
+ normalDims.removeAll(jointDims);
+
+ int normalDimSize = normalDims.size();
+ int hierarchySize = (agg.getSelectRule().hierarchy_dims == null ? 0 : agg.getSelectRule().hierarchy_dims.length);
+ int jointSize = agg.getSelectRule().joint_dims == null ? 0 : agg.getSelectRule().joint_dims.length;
+ if (normalDimSize + hierarchySize + jointSize > maxSize) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions");
+ }
+
+ if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+ }
+ if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims");
+ }
+
+ int jointDimNum = 0;
+ if (agg.getSelectRule().joint_dims != null) {
+ for (String[] joints : agg.getSelectRule().joint_dims) {
+
+ Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ for (String s : joints) {
+ oneJoint.add(s);
+ }
+
+ if (oneJoint.size() < 2) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint");
+ }
+ jointDimNum += oneJoint.size();
+
+ int overlapHierarchies = 0;
+ if (agg.getSelectRule().hierarchy_dims != null) {
+ for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) {
+ Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy)));
+
+ if (!share.isEmpty()) {
+ overlapHierarchies++;
+ }
+ if (share.size() > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy");
+ }
+ }
+
+ if (overlapHierarchies > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies");
+ }
+ }
+ }
+
+ if (jointDimNum != jointDims.size()) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint");
+ }
+ }
+
+ index++;
+ }
+ }
+
+ protected int getMaxAgrGroupSize() {
+ String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
+ return Integer.parseInt(size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index e37b9a9..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- *
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- innerValidateMaxSize(cube, context);
- }
-
- /**
- * @param cube
- * @param context
- */
- private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
- int maxSize = getMaxAgrGroupSize();
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- if (group.length >= maxSize) {
- context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
- }
- }
- }
-
- protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
index 52e5b24..80d4676 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
@@ -20,13 +20,9 @@ package org.apache.kylin.cube.model.validation.rule;
import org.apache.kylin.metadata.MetadataConstants;
-/**
- * @author jianliu
- *
- */
public interface IKylinValidationConstants extends MetadataConstants {
- public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
+ public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 12;
public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index 4585900..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- *
- * @author jianliu
- *
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- Set<String> mands = new HashSet<String>();
- RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
- if (cols == null || cols.length == 0) {
- return;
- }
- for (int i = 0; i < cols.length; i++) {
- RowKeyColDesc rowKeyColDesc = cols[i];
- if (rowKeyColDesc.isMandatory()) {
- mands.add(rowKeyColDesc.getColumn());
- }
- }
- if (mands.isEmpty()) {
- return;
- }
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- for (int j = 0; j < group.length; j++) {
- String col = group[j];
- if (mands.contains(col)) {
- context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
- }
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
index e90f6cd..7f4b6e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
@@ -18,7 +18,6 @@
package org.apache.kylin.cube.model.validation.rule;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
@@ -28,9 +27,6 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
/**
* Validate that only one of "length" and "dictionary" appears on rowkey_column
- *
- * @author jianliu
- *
*/
public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
@@ -61,12 +57,12 @@ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
for (int i = 0; i < rcd.length; i++) {
RowKeyColDesc rd = rcd[i];
- if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
- }
- if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
- }
+ // if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
+ // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
+ // }
+ // if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
+ // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
+ // }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
new file mode 100644
index 0000000..772da17
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.cube.model.v2.CubeDesc;
+import org.apache.kylin.cube.model.v2.DimensionDesc;
+import org.apache.kylin.cube.model.v2.HBaseMappingDesc;
+import org.apache.kylin.cube.model.v2.RowKeyColDesc;
+import org.apache.kylin.cube.model.v2.RowKeyDesc;
+
+import com.google.common.collect.Lists;
+
+public class CubeDescUpgraderV2 {
+
+ @SuppressWarnings("unused")
+ private static final Log logger = LogFactory.getLog(CubeDescUpgraderV2.class);
+ private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
+
+ private String resourcePath;
+
+ private List<String[]> oldHierarchies = Lists.newArrayList();
+ private List<String> oldMandatories = Lists.newArrayList();
+ private String[][] oldAggGroup = null;
+ private Set<String> allRowKeyCols = newIgnoreCaseSet(null);
+
+ public CubeDescUpgraderV2(String resourcePath) {
+ this.resourcePath = resourcePath;
+ }
+
+ public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
+ CubeDesc oldModel = loadOldCubeDesc(resourcePath);
+
+ org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
+ copyUnChangedProperties(oldModel, newModel);
+ upgradeDimension(oldModel, newModel);
+ upgradeRowKey(oldModel, newModel);
+ upgradeHBaseMapping(oldModel, newModel);
+ upgradeAggregationGroup(newModel);//must do at last
+
+ return newModel;
+ }
+
+ private CubeDesc loadOldCubeDesc(String path) throws IOException {
+ ResourceStore store = getStore();
+
+ CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer);
+
+ if (StringUtils.isBlank(ndesc.getName())) {
+ throw new IllegalStateException("CubeDesc name must not be blank");
+ }
+
+ return ndesc;
+ }
+
+ private Set<String> newIgnoreCaseSet(Set<String> input) {
+ Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (input != null)
+ ret.addAll(input);
+ return ret;
+ }
+
+ private String[] toArray(Set<String> input) {
+ return input.toArray(new String[input.size()]);
+ }
+
+ private boolean rowKeyColExistsInMultipleAggGroup() {
+ if (oldAggGroup == null)
+ return false;
+
+ int total = 0;
+ Set<String> overall = newIgnoreCaseSet(null);
+ for (String[] group : oldAggGroup) {
+ Set<String> temp = newIgnoreCaseSet(null);
+ for (String entry : group) {
+
+ overall.add(entry);
+ temp.add(entry);
+ }
+ total += temp.size();
+ }
+ return overall.size() != total;
+ }
+
+ private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) {
+
+ List<AggregationGroup> aggs = Lists.newArrayList();
+ if (oldAggGroup == null || oldAggGroup.length == 0 || //
+ ((oldAggGroup.length == 1) && (oldAggGroup[0].length == allRowKeyCols.size()))) {
+ AggregationGroup agg = new AggregationGroup();
+ agg.setIncludes(toArray(allRowKeyCols));
+ agg.setSelectRule(null);
+ aggs.add(agg);
+ } else {
+ if (rowKeyColExistsInMultipleAggGroup()) {
+ throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!");
+ }
+
+ Set<String> visited = newIgnoreCaseSet(null);
+
+ for (String[] group : oldAggGroup) {
+ AggregationGroup agg = new AggregationGroup();
+
+ Set<String> remaining = newIgnoreCaseSet(allRowKeyCols);
+ remaining.removeAll(visited);
+
+ Set<String> joint = newIgnoreCaseSet(remaining);
+ joint.removeAll(oldMandatories);
+
+ Set<String> groupAsSet = newIgnoreCaseSet(null);
+ for (String entry : group) {
+ groupAsSet.add(entry);
+ }
+ visited.addAll(groupAsSet);
+ joint.removeAll(groupAsSet);
+
+ List<String> mandatories = Lists.newArrayList();
+ List<String[]> hierarchies = Lists.newArrayList();
+
+ for (String s : oldMandatories) {
+ mandatories.add(s);
+ }
+
+ for (String[] h : oldHierarchies) {
+ if (groupAsSet.containsAll(Arrays.asList(h))) {
+ hierarchies.add(h);
+ }
+ }
+
+ agg.setIncludes(toArray(remaining));
+
+ SelectRule selectRule = new SelectRule();
+ selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]);
+ if (joint.size() != 0) {
+ selectRule.joint_dims = new String[1][];
+ selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]);
+ } else {
+ selectRule.joint_dims = new String[0][];
+ }
+ selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]);
+ agg.setSelectRule(selectRule);
+
+ aggs.add(agg);
+
+ }
+ }
+ newModel.setAggregationGroups(aggs);
+ }
+
+ private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ List<DimensionDesc> oldDimensions = oldModel.getDimensions();
+ if (oldDimensions == null) {
+ throw new IllegalArgumentException("dimensions is null");
+ }
+ List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList();
+
+ for (DimensionDesc oldDim : oldDimensions) {
+ if (oldDim.isDerived()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn("{FK}");
+ newDim.setDerived(oldDim.getDerived());
+
+ newDimensions.add(newDim);
+ } else {
+ if (oldDim.isHierarchy()) {
+ oldHierarchies.add(oldDim.getColumn());
+ }
+
+ for (String columnStr : oldDim.getColumn()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn(columnStr);
+ newDim.setDerived(null);
+
+ newDimensions.add(newDim);
+ }
+ }
+ }
+
+ newModel.setDimensions(newDimensions);
+ }
+
+ private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ RowKeyDesc oldRowKey = oldModel.getRowkey();
+ if (oldModel == null) {
+ throw new IllegalArgumentException("RowKeyDesc is null");
+ }
+
+ if (oldRowKey.getRowKeyColumns() == null) {
+ throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null");
+ }
+
+ org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc();
+ org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length];
+ int index = 0;
+ for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) {
+ org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc();
+
+ allRowKeyCols.add(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.isMandatory()) {
+ oldMandatories.add(oldRowKeyCol.getColumn());
+ }
+
+ newRowKeyCol.setColumn(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) {
+ newRowKeyCol.setEncoding("dict");
+ } else if (oldRowKeyCol.getLength() > 0) {
+ newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength());
+ } else {
+ throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength());
+ }
+ cols[index++] = newRowKeyCol;
+ }
+ oldAggGroup = oldRowKey.getAggregationGroups();
+
+ newRowKey.setRowkeyColumns(cols);
+ newModel.setRowkey(newRowKey);
+ }
+
+ private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping();
+ try {
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, hbaseMappingDesc);
+ byte[] blob = os.toByteArray();
+ ByteArrayInputStream is = new ByteArrayInputStream(blob);
+ org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class);
+ newModel.setHbaseMapping(newHBaseMappingDesc);
+
+ } catch (IOException e) {
+ throw new RuntimeException("error when copying HBaseMappingDesc");
+ }
+ }
+
+ private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ newModel.setUuid(oldModel.getUuid());
+ newModel.setName(oldModel.getName());
+ newModel.setDescription(oldModel.getDescription());
+ newModel.setMeasures(oldModel.getMeasures());
+ newModel.setNullStrings(oldModel.getNullStrings());
+ newModel.setModelName(oldModel.getModelName());
+ newModel.setNotifyList(oldModel.getNotifyList());
+ newModel.setLastModified(oldModel.getLastModified());
+ newModel.setStorageType(oldModel.getStorageType());
+ newModel.setEngineType(oldModel.getEngineType());
+ }
+
+ protected static ResourceStore getStore() {
+ return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
new file mode 100644
index 0000000..4d3fce9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+
+/**
+ * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1,
+ * that upgrades metadata store from v1(prior kylin 0.7) to v2.
+ * the major difference is that we split cube desc to cube desc + model desc
+ *
+ * this CubeMetadataUpgradeV2 upgrades metadata store from v2(prior kylin 2.1) to v3
+ * the major different is a brand new definition of partial cubes to allow users to select
+ * cuboids more flexibly
+ */
+public class CubeMetadataUpgradeV2 {
+
+ private KylinConfig config = null;
+ private ResourceStore store;
+
+ private List<String> updatedResources = Lists.newArrayList();
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV2.class);
+
+ public CubeMetadataUpgradeV2(String newMetadataUrl) {
+ KylinConfig.destoryInstance();
+ System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl);
+ KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl);
+
+ config = KylinConfig.getInstanceFromEnv();
+ store = getStore();
+ }
+
+ public void upgrade() {
+
+ upgradeCubeDesc();
+ verify();
+ }
+
+ public void verify() {
+ MetadataManager.clearCache();
+ MetadataManager.getInstance(config);
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.clearCache();
+ CubeManager.getInstance(config);
+ ProjectManager.clearCache();
+ ProjectManager.getInstance(config);
+ //cleanup();
+ }
+
+ private List<String> listResourceStore(String pathRoot) {
+ List<String> paths = null;
+ try {
+ paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+ }
+
+ return paths;
+ }
+
+ private void upgradeCubeDesc() {
+ logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+
+ List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+ for (String path : paths) {
+
+ try {
+ CubeDescUpgraderV2 upgrade = new CubeDescUpgraderV2(path);
+ CubeDesc ndesc = upgrade.upgrade();
+ ndesc.setSignature(ndesc.calculateSignature());
+
+ getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+ updatedResources.add(ndesc.getResourcePath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
+ }
+ }
+
+ }
+
+ private ResourceStore getStore() {
+ return ResourceStore.getStore(config);
+ }
+
+ public static void main(String[] args) {
+
+ if (!(args != null && (args.length == 1 || args.length == 2))) {
+ System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
+ return;
+ }
+
+ String exportFolder = args[0];
+ boolean verify = false;
+ if (args.length == 2 && "verify".equals(args[1])) {
+ System.out.println("Only verify the metadata in folder " + exportFolder);
+ verify = true;
+ }
+
+ CubeMetadataUpgradeV2 instance = null;
+ if (verify) {
+ instance = new CubeMetadataUpgradeV2(exportFolder);
+ instance.verify();
+ } else {
+ File oldMetaFolder = new File(exportFolder);
+ if (!oldMetaFolder.exists()) {
+ System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
+ return;
+ }
+
+ if (!oldMetaFolder.isDirectory()) {
+ System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
+ return;
+ }
+
+ String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v3";//upgrades metadata store to v3 format
+ try {
+ FileUtils.deleteDirectory(new File(newMetadataUrl));
+ FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ instance = new CubeMetadataUpgradeV2(newMetadataUrl);
+ instance.upgrade();
+ logger.info("=================================================================");
+ logger.info("Run CubeMetadataUpgrade completed;");
+
+ }
+
+ logger.info("=================================================================");
+ if (instance.errorMsgs.size() > 0) {
+ logger.info("Here are the error/warning messages, you may need check:");
+ for (String s : instance.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The migration is success.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 0cfd020..ada31b6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -34,7 +34,6 @@
package org.apache.kylin.cube.util;
import java.io.IOException;
-import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -42,14 +41,6 @@ import java.util.Map;
import javax.annotation.Nullable;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
@@ -59,12 +50,25 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
/**
*/
public class CubingUtils {
@@ -82,8 +86,7 @@ public class CubingUtils {
@Nullable
@Override
public Integer[] apply(@Nullable Long cuboidId) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] result = new Integer[bitSet.cardinality()];
+ Integer[] result = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
@@ -100,8 +103,7 @@ public class CubingUtils {
final Map<Long, HyperLogLogPlusCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
result.put(cuboidId, new HyperLogLogPlusCounter(14));
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+ Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
new file mode 100644
index 0000000..6cfaaa4
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author jianliu
+ *
+ */
+public class AggregationGroupRuleTest {
+
+ private CubeDesc cube;
+ private ValidateContext vContext = new ValidateContext();
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
+ this.cube = desc2;
+
+ }
+
+ @Test
+ public void testOneMandatoryColumn() {
+ IValidatorRule<CubeDesc> rule = new AggregationGroupRule() {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+ * #getMaxAgrGroupSize()
+ */
+ @Override
+ protected int getMaxAgrGroupSize() {
+ return 3;
+ }
+ };
+ rule.validate(cube, vContext);
+ vContext.print(System.out);
+ assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
+ assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
+ assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[2].getMessage()
+ // .startsWith("Hierachy column"));
+ }
+
+ @Test
+ public void testAggColumnSize() {
+ AggregationGroupRule rule = new AggregationGroupRule() {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+ * #getMaxAgrGroupSize()
+ */
+ @Override
+ protected int getMaxAgrGroupSize() {
+ return 20;
+ }
+ };
+ rule.validate(cube, vContext);
+ vContext.print(System.out);
+ assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[0].getMessage()
+ // .startsWith("Aggregation group"));
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[0].getMessage()
+ // .startsWith("Hierachy column"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index e7914c7..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 3;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[2].getMessage()
- // .startsWith("Hierachy column"));
- }
-
- @Test
- public void testAggColumnSize() {
- AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 20;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Aggregation group"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Hierachy column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index d7feb56..0de5c22 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -56,10 +56,10 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
- DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+ DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
System.out.println(JsonUtil.writeValueAsIndentString(info1));
- DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+ DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
System.out.println(JsonUtil.writeValueAsIndentString(info2));
assertTrue(info1.getUuid() == info2.getUuid());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
deleted file mode 100644
index 29b7183..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class MandatoryColumnRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
- rule.validate(cube, vContext);
- assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 83e8d94..b45a005 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -83,34 +83,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
}
- @Test
- public void testFindSmallerSibling1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(255);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(133);
- assertEquals("[131]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(127);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(134);
- assertEquals("[131, 133]", sortToString(siblings));
-
- siblings = scheduler.findSmallerSibling(130);
- assertEquals("[129]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(5);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(135);
- assertEquals("[]", siblings.toString());
- }
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
for (long cuboidId : cuboidIds) {
@@ -123,31 +95,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
}
- @Test
- public void testFindSmallerSibling2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(511);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110111111"));
- assertEquals("[383]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("101110111"));
- assertEquals("[319]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110000000"));
- assertEquals("[288, 320]", sortToString(siblings));
- }
@Test
public void testGetSpanningCuboid2() {
@@ -234,19 +181,19 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidGeneration1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
public void testCuboidGeneration2() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
public void testCuboidGeneration3() {
CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
@@ -266,6 +213,11 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
}
+ @Test
+ public void testPotentialChild() {
+ assertEquals(1, 2);
+ }
+
private String sortToString(Collection<Long> longs) {
ArrayList<Long> copy = new ArrayList<Long>(longs);
Collections.sort(copy);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
index e37aabc..774ad09 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
@@ -100,7 +100,7 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
Assert.assertTrue(dims.size() > 0);
for (DimensionDesc dim : dims) {
- Assert.assertTrue(dim.getColumn().length > 0);
+ Assert.assertTrue(dim.getColumn() != null);
}
Assert.assertTrue(cubedesc1.getMeasures().size() > 0);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..2a1304a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -223,11 +223,11 @@ public class DictionaryManager {
}
}
- public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, boolean usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
logger.info("building dictionary for " + col);
- TblColRef srcCol = decideSourceData(model, dict, col);
+ TblColRef srcCol = decideSourceData(model, usingDict, col);
String srcTable = srcCol.getTable();
String srcColName = srcCol.getName();
int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
@@ -259,9 +259,9 @@ public class DictionaryManager {
/**
* Decide a dictionary's source data, leverage PK-FK relationship.
*/
- public TblColRef decideSourceData(DataModelDesc model, String dict, TblColRef col) throws IOException {
+ public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, TblColRef col) throws IOException {
// Note FK on fact table is supported by scan the related PK on lookup table
- if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) {
+ if (usingDict) {
// FK on fact table and join type is inner, use PK from lookup instead
if (model.isFactTable(col.getTable())) {
TblColRef pkCol = model.findPKByFK(col, "inner");
@@ -270,7 +270,7 @@ public class DictionaryManager {
}
return col;
} else
- throw new IllegalArgumentException("Unknown dictionary value: " + dict);
+ throw new IllegalArgumentException("Not using Dictionary ");
}
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 1561b1f..de471c1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -28,8 +28,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class MeasureDesc {
- @JsonProperty("id")
- private int id;
@JsonProperty("name")
private String name;
@JsonProperty("function")
@@ -37,14 +35,6 @@ public class MeasureDesc {
@JsonProperty("dependent_measure_ref")
private String dependentMeasureRef;
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
public String getName() {
return name;
}
@@ -91,11 +81,6 @@ public class MeasureDesc {
}
@Override
- public int hashCode() {
- return id;
- }
-
- @Override
public String toString() {
return "MeasureDesc [name=" + name + ", function=" + function + "]";
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 12cac94..3c10c09 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -63,7 +63,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 3: Build Cube
RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
- final int groupRowkeyColumnsCount = rowKeyDesc.getNCuboidBuildLevels();
+ final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
// base cuboid step
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index b97c88a..b656da2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -62,7 +62,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
for (int i = 0; i < dictionaryColumns.size(); i++) {
TblColRef col = dictionaryColumns.get(i);
- String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+ String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
if (cubeDesc.getModel().isFactTable(scanTable)) {
factDictCols.add(i);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index e43d5d1..0c5b9b3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -89,8 +89,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) {
allCuboids.add(cuboidId);
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] indice = new Integer[bitSet.cardinality()];
+ Integer[] indice = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index 6098381..8416d95 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -88,7 +88,7 @@ public class MapContextGTRecordWriter implements ICuboidWriter {
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
keyBuf = rowKeyEncoder.createBuf();
- dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+ dimensions = Long.bitCount(cuboidId);
measureColumnsIndex = new int[measureCount];
for (int i = 0; i < measureCount; i++) {
measureColumnsIndex[i] = dimensions + i;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 286ff02..53c23b8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -99,7 +99,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 68d1481..ea1fbb0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index b73fda4..8fced61 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -112,7 +112,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
CubeDesc cubeDesc = cube.getDescriptor();
for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
- String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+ String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(),true, col).getTable();
if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
colsNeedMeringDict.add(col);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fc016fc..410cec7 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -77,8 +77,7 @@ public class CubeSamplingTest {
}
private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] indice = new Integer[bitSet.cardinality()];
+ Integer[] indice = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 01d97fd..32d4d2a 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -229,8 +229,7 @@ public class SparkCubing extends AbstractApplication {
final ByteArray[] row_hashcodes = new ByteArray[nRowKey];
for (Long cuboidId : allCuboidIds) {
- BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
- Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+ Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
@@ -480,7 +479,7 @@ public class SparkCubing extends AbstractApplication {
final Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fs = FileSystem.get(hbaseConf);
FsPermission permission = new FsPermission((short) 0777);
- for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) {
String cfName = cf.getName();
Path columnFamilyPath = new Path(hfileLocation, cfName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 6b3a82c..9c1d681 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -18,7 +18,6 @@
package org.apache.kylin.engine.spark.cube;
import java.nio.ByteBuffer;
-import java.util.BitSet;
import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
@@ -70,7 +69,7 @@ public final class DefaultTupleConverter implements TupleConverter {
Cuboid cuboid = Cuboid.findById(segment.getCubeDesc(), cuboidId);
RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
- final int dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+ final int dimensions = Long.bitCount(cuboidId);
int[] measureColumnsIndex = getMeasureColumnsIndex();
for (int i = 0; i < measureCount; i++) {
measureColumnsIndex[i] = dimensions + i;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/examples/test_case_data/localmeta/cube_desc/sample.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/sample.json b/examples/test_case_data/localmeta/cube_desc/sample.json
new file mode 100644
index 0000000..9581b70
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/sample.json
@@ -0,0 +1,257 @@
+{
+ "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+ "name": "test_kylin_cube_with_slr_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": null,
+ "derived": [
+ "WEEK_BEG_DT"
+ ],
+ },
+ {
+ "id": 1,
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 2,
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ],
+ "derived": null,
+ "hierarchy": true
+ },
+ {
+ "id": 3,
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": [
+ "LSTG_FORMAT_NAME"
+ ],
+ "derived": null,
+ "hierarchy": false
+ },
+ {
+ "id": 4,
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": null,
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 5,
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": null,
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 6,
+ "name": "SELLER_ID",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": [
+ "SELLER_ID"
+ ],
+ "derived": null,
+ "hierarchy": false
+ }
+ ],
+ "measures": [
+ {
+ "id": 1,
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 2,
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 3,
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 4,
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1"
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 5,
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT"
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "seller_id",
+ "length": 18,
+ "dictionary": null,
+ "mandatory": true
+ },
+ {
+ "column": "cal_dt",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "leaf_categ_id",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "meta_categ_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "categ_lvl2_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "categ_lvl3_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "lstg_format_name",
+ "length": 12,
+ "dictionary": null,
+ "mandatory": false
+ },
+ {
+ "column": "lstg_site_id",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "slr_segment_cd",
+ }
+ ],
+ "aggregation_groups": [
+ {
+ "include": [
+ "leaf_categ_id",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "cal_dt"
+ ],
+ "selective_rules": {
+ "hierarchy_dims": [
+ [
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ ],
+ "mandatory_dims": [
+ "mandatory"
+ ],
+ "joint_dims": [
+ [
+ "leaf_categ_id",
+ "cal_dt"
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "last_modified": 1422435345330,
+ "model_name": "test_kylin_inner_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "notify_list": null,
+ "engine_type": 2,
+ "storage_type": 2
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 6670de1..28b0b29 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -194,7 +194,7 @@ public class CubeService extends BasicService {
}
try {
- int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc);
+ int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc,false);
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
} catch (Exception e) {
getCubeDescManager().removeCubeDesc(createdDesc);
@@ -260,7 +260,7 @@ public class CubeService extends BasicService {
}
CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
- int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
+ int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc,false);
logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");
ProjectManager projectManager = getProjectManager();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index c313cc3..be6e22c 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -89,7 +89,7 @@ public class CubeControllerTest extends ServiceTestBase {
newCube.setModelName(cube.getModelName());
newCube.setModel(cube.getModel());
newCube.setDimensions(cube.getDimensions());
- newCube.setHBaseMapping(cube.getHBaseMapping());
+ newCube.setHbaseMapping(cube.getHbaseMapping());
newCube.setMeasures(cube.getMeasures());
newCube.setConfig(cube.getConfig());
newCube.setRowkey(cube.getRowkey());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index f84e4e6..fcfc335 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -154,7 +154,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<HBaseKeyRange> scans = buildScanRanges(flattenToOrAndFilter(filterD), dimensionsD);
// check involved measures, build value decoder for each each family:column
- List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context);
+ List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHbaseMapping(), metrics, context);
//memory hungry distinct count are pushed down to coprocessor, no need to set threshold any more
//setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 8813901..6a15bc3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -12,7 +12,6 @@ import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -21,12 +20,9 @@ import org.apache.kylin.cube.kv.RowKeyDecoder;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -104,7 +100,7 @@ public class CubeTupleConverter {
}
// prepare derived columns and filler
- Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(dimCols, null);
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
TblColRef[] hostCols = entry.getKey().data;
for (DeriveInfo deriveInfo : entry.getValue()) {
[5/7] incubator-kylin git commit: whitelist
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index abec436..87b182d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,127 +18,51 @@
package org.apache.kylin.cube.model;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import java.util.Set;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class RowKeyDesc {
- public static class HierarchyMask {
- public long fullMask;
- public long[] allMasks;
- }
-
- public static class AggrGroupMask {
- public AggrGroupMask(int size) {
- groupOneBitMasks = new long[size];
- }
-
- public long groupMask;
- public long groupOneBitMasks[];
- public long uniqueMask;
- public long leftoverMask;
- }
-
@JsonProperty("rowkey_columns")
private RowKeyColDesc[] rowkeyColumns;
- @JsonProperty("aggregation_groups")
- private String[][] aggregationGroups;
// computed content
+ private long fullMask;
private CubeDesc cubeDesc;
private Map<TblColRef, RowKeyColDesc> columnMap;
- private long fullMask;
- private long mandatoryColumnMask;
- private AggrGroupMask[] aggrGroupMasks;
- private long aggrGroupFullMask;
- private long tailMask;
-
- private List<HierarchyMask> hierarchyMasks;
-
public RowKeyColDesc[] getRowKeyColumns() {
return rowkeyColumns;
}
- // search a specific row key col
- public int getRowKeyIndexByColumnName(String columnName) {
- if (this.rowkeyColumns == null)
- return -1;
-
- for (int i = 0; i < this.rowkeyColumns.length; ++i) {
- RowKeyColDesc desc = this.rowkeyColumns[i];
- if (desc.getColumn().equalsIgnoreCase(columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- public int getNCuboidBuildLevels() {
- // N aggregation columns requires N levels of cuboid build
- // - N columns requires N-1 levels build
- // - zero tail cuboid needs one more additional level
- Set<String> aggDims = new HashSet<String>();
- for (String[] aggrGroup : aggregationGroups) {
- for (String dim : aggrGroup) {
- aggDims.add(dim);
- }
- }
- return aggDims.size();
- }
-
- public String[][] getAggregationGroups() {
- return aggregationGroups;
- }
-
- public CubeDesc getCubeRef() {
- return cubeDesc;
- }
-
- public void setCubeRef(CubeDesc cubeRef) {
+ // public int getNCuboidBuildLevels() {
+ // // N aggregation columns requires N levels of cuboid build
+ // // - N columns requires N-1 levels build
+ // // - zero tail cuboid needs one more additional level
+ // Set<String> aggDims = new HashSet<String>();
+ // for (String[] aggrGroup : aggregationGroups) {
+ // for (String dim : aggrGroup) {
+ // aggDims.add(dim);
+ // }
+ // }
+ // return aggDims.size();
+ // }
+
+ public void setCubeDesc(CubeDesc cubeRef) {
this.cubeDesc = cubeRef;
}
- public long getFullMask() {
- return fullMask;
- }
-
- public long getMandatoryColumnMask() {
- return mandatoryColumnMask;
- }
-
- public long getAggrGroupFullMask() {
- return aggrGroupFullMask;
- }
-
- public AggrGroupMask[] getAggrGroupMasks() {
- return aggrGroupMasks;
- }
-
- public List<HierarchyMask> getHierarchyMasks() {
- return hierarchyMasks;
- }
-
- public long getTailMask() {
- return tailMask;
- }
-
public int getColumnBitIndex(TblColRef col) {
return getColDesc(col).getBitIndex();
}
@@ -147,10 +71,6 @@ public class RowKeyDesc {
return getColDesc(col).getLength();
}
- public String getDictionary(TblColRef col) {
- return getColDesc(col).getDictionary();
- }
-
private RowKeyColDesc getColDesc(TblColRef col) {
RowKeyColDesc desc = columnMap.get(col);
if (desc == null)
@@ -158,40 +78,37 @@ public class RowKeyDesc {
return desc;
}
- public boolean isUseDictionary(int index) {
- String useDictionary = rowkeyColumns[index].getDictionary();
- return useDictionary(useDictionary);
- }
-
public boolean isUseDictionary(TblColRef col) {
- String useDictionary = getDictionary(col);
- return useDictionary(useDictionary);
+ return getColDesc(col).isUsingDictionary();
}
- private boolean useDictionary(String useDictionary) {
- return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary);
- }
+ public void init(CubeDesc cubeDesc) {
- public void init(CubeDesc cube) {
- setCubeRef(cube);
- Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation();
+ setCubeDesc(cubeDesc);
+ Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation();
buildRowKey(colNameAbbr);
- buildAggregationGroups(colNameAbbr);
- buildHierarchyMasks();
+ }
+
+ public RowKeyColDesc[] getRowkeyColumns() {
+ return rowkeyColumns;
+ }
+
+ public void setRowkeyColumns(RowKeyColDesc[] rowkeyColumns) {
+ this.rowkeyColumns = rowkeyColumns;
}
@Override
public String toString() {
- return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]";
+ return Objects.toStringHelper(this).add("RowKeyColumns", Arrays.toString(rowkeyColumns)).toString();
}
private void buildRowKey(Map<String, TblColRef> colNameAbbr) {
columnMap = new HashMap<TblColRef, RowKeyColDesc>();
- mandatoryColumnMask = 0;
for (int i = 0; i < rowkeyColumns.length; i++) {
RowKeyColDesc rowKeyColDesc = rowkeyColumns[i];
+ rowKeyColDesc.init();
String column = rowKeyColDesc.getColumn();
rowKeyColDesc.setColumn(column.toUpperCase());
rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1);
@@ -201,95 +118,17 @@ public class RowKeyDesc {
}
columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc);
-
- if (rowKeyColDesc.isMandatory()) {
- mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex();
- }
- }
- }
-
- private void buildAggregationGroups(Map<String, TblColRef> colNameAbbr) {
- if (aggregationGroups == null) {
- aggregationGroups = new String[0][];
- }
-
- for (int i = 0; i < aggregationGroups.length; i++) {
- StringUtil.toUpperCaseArray(aggregationGroups[i], this.aggregationGroups[i]);
}
+ this.fullMask = 0L;
for (int i = 0; i < this.rowkeyColumns.length; i++) {
int index = rowkeyColumns[i].getBitIndex();
this.fullMask |= 1L << index;
}
-
- this.aggrGroupMasks = new AggrGroupMask[aggregationGroups.length];
- for (int i = 0; i < this.aggregationGroups.length; i++) {
- String[] aggGrp = this.aggregationGroups[i];
- AggrGroupMask mask = new AggrGroupMask(aggGrp.length);
-
- for (int j = 0; j < aggGrp.length; j++) {
- TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase());
- if (aggCol == null) {
- throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName());
- }
- Integer index = getColumnBitIndex(aggCol);
- mask.groupMask |= 1L << index;
- mask.groupOneBitMasks[j] = 1L << index;
- this.aggrGroupFullMask |= 1L << index;
- }
- this.aggrGroupMasks[i] = mask;
- }
-
- this.tailMask = fullMask ^ mandatoryColumnMask ^ aggrGroupFullMask;
-
- // unique mask = (bits in this group) - (bits in following groups)
- // leftover mask = (tail bits) + (bits in following groups) - (bits in
- // this group)
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- AggrGroupMask mask = aggrGroupMasks[i];
-
- mask.uniqueMask = mask.groupMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.uniqueMask &= ~aggrGroupMasks[j].groupMask;
- }
-
- mask.leftoverMask = tailMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.leftoverMask |= aggrGroupMasks[j].groupMask;
- }
- mask.leftoverMask &= ~mask.groupMask;
- }
}
- private void buildHierarchyMasks() {
- this.hierarchyMasks = new ArrayList<HierarchyMask>();
-
- for (DimensionDesc dimension : this.cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- if (hierarchies == null || hierarchies.length == 0)
- continue;
-
- HierarchyMask mask = new HierarchyMask();
- ArrayList<Long> allMaskList = new ArrayList<Long>();
- for (int i = 0; i < hierarchies.length; i++) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((tailMask & bit) > 0)
- continue; // ignore levels in tail, they don't participate
- // aggregation group combination anyway
-
- mask.fullMask |= bit;
- allMaskList.add(mask.fullMask);
- }
-
- mask.allMasks = new long[allMaskList.size()];
- for (int i = 0; i < allMaskList.size(); i++)
- mask.allMasks[i] = allMaskList.get(i);
-
- this.hierarchyMasks.add(mask);
- }
+ public long getFullMask() {
+ return this.fullMask;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
new file mode 100644
index 0000000..63b0fc4
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class SelectRule {
+ @JsonProperty("hierarchy_dims")
+ public String[][] hierarchy_dims;
+ @JsonProperty("mandatory_dims")
+ public String[] mandatory_dims;
+ @JsonProperty("joint_dims")
+ public String[][] joint_dims;
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeBuildTypeEnum.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeBuildTypeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeBuildTypeEnum.java
new file mode 100644
index 0000000..2544913
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeBuildTypeEnum.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+/**
+ * @author xduo
+ *
+ */
+public enum CubeBuildTypeEnum {
+ /**
+ * rebuild a segment or incremental build
+ */
+ BUILD,
+ /**
+ * merge segments
+ */
+ MERGE,
+
+ /**
+ * refresh segments
+ */
+ REFRESH
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
new file mode 100644
index 0000000..04e4c65
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
@@ -0,0 +1,867 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.CaseInsensitiveStringMap;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IEngineAware;
+import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ */
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class CubeDesc extends RootPersistentEntity {
+
+ public static enum DeriveType {
+ LOOKUP, PK_FK
+ }
+
+ public static class DeriveInfo {
+ public DeriveType type;
+ public DimensionDesc dimension;
+ public TblColRef[] columns;
+ public boolean isOneToOne; // only used when ref from derived to host
+
+ DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
+ this.type = type;
+ this.dimension = dimension;
+ this.columns = columns;
+ this.isOneToOne = isOneToOne;
+ }
+
+ @Override
+ public String toString() {
+ return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
+ }
+
+ }
+
+ private KylinConfig config;
+ private DataModelDesc model;
+
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("model_name")
+ private String modelName;
+ @JsonProperty("description")
+ private String description;
+ @JsonProperty("null_string")
+ private String[] nullStrings;
+ @JsonProperty("dimensions")
+ private List<DimensionDesc> dimensions;
+ @JsonProperty("measures")
+ private List<MeasureDesc> measures;
+ @JsonProperty("rowkey")
+ private RowKeyDesc rowkey;
+ @JsonProperty("hbase_mapping")
+ private HBaseMappingDesc hbaseMapping;
+ @JsonProperty("signature")
+ private String signature;
+ @JsonProperty("notify_list")
+ private List<String> notifyList;
+ @JsonProperty("status_need_notify")
+ private List<String> statusNeedNotify = Collections.emptyList();
+ @JsonProperty("auto_merge_time_ranges")
+ private long[] autoMergeTimeRanges;
+ @JsonProperty("retention_range")
+ private long retentionRange = 0;
+
+ @JsonProperty("engine_type")
+ private int engineType = IEngineAware.ID_MR_V1;
+ @JsonProperty("storage_type")
+ private int storageType = IStorageAware.ID_HBASE;
+
+ private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
+ private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
+ private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
+
+ private LinkedHashSet<TblColRef> measureDisplayColumns = new LinkedHashSet<TblColRef>();
+ private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
+ private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
+
+ public boolean isEnableSharding() {
+ //in the future may extend to other storage that is shard-able
+ return storageType == IStorageAware.ID_SHARDED_HBASE;
+ }
+
+ /**
+ * Error messages during resolving json metadata
+ */
+ private List<String> errors = new ArrayList<String>();
+
+ /**
+ * @return all columns this cube can support, including derived
+ */
+ public Set<TblColRef> listAllColumns() {
+ return allColumns;
+ }
+
+ /**
+ * @return dimension columns including derived, BUT NOT measures
+ */
+ public Set<TblColRef> listDimensionColumnsIncludingDerived() {
+ return dimensionColumns;
+ }
+
+ /**
+ * @return dimension columns excluding derived and measures
+ */
+ public List<TblColRef> listDimensionColumnsExcludingDerived() {
+ List<TblColRef> result = new ArrayList<TblColRef>();
+ for (TblColRef col : dimensionColumns) {
+ if (isDerived(col) == false)
+ result.add(col);
+ }
+ return result;
+ }
+
+ /**
+ * Find FunctionDesc by Full Expression.
+ *
+ * @return
+ */
+ public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
+ for (MeasureDesc m : measures) {
+ if (m.getFunction().equals(manualFunc))
+ return m.getFunction();
+ }
+ return null;
+ }
+
+ public TblColRef findColumnRef(String table, String column) {
+ Map<String, TblColRef> cols = columnMap.get(table);
+ if (cols == null)
+ return null;
+ else
+ return cols.get(column);
+ }
+
+ public DimensionDesc findDimensionByColumn(TblColRef col) {
+ for (DimensionDesc dim : dimensions) {
+ if (ArrayUtils.contains(dim.getColumnRefs(), col))
+ return dim;
+ }
+ return null;
+ }
+
+ public DimensionDesc findDimensionByTable(String lookupTableName) {
+ lookupTableName = lookupTableName.toUpperCase();
+ for (DimensionDesc dim : dimensions)
+ if (dim.getTable() != null && dim.getTable().equals(lookupTableName))
+ return dim;
+ return null;
+ }
+
+ public DimensionDesc findDimensionByName(String dimName) {
+ dimName = dimName.toUpperCase();
+ for (DimensionDesc dim : dimensions) {
+ if (dimName.equals(dim.getName()))
+ return dim;
+ }
+ return null;
+ }
+
+ /**
+ * Get all functions from each measure.
+ *
+ * @return
+ */
+ public List<FunctionDesc> listAllFunctions() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
+ for (MeasureDesc m : measures) {
+ functions.add(m.getFunction());
+ }
+ return functions;
+ }
+
+ public boolean isDerived(TblColRef col) {
+ return derivedToHostMap.containsKey(col);
+ }
+
+ public DeriveInfo getHostInfo(TblColRef derived) {
+ return derivedToHostMap.get(derived);
+ }
+
+ public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
+ Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
+ for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
+ Array<TblColRef> hostCols = entry.getKey();
+ boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
+ if (!hostOnRow)
+ continue;
+
+ List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
+ for (DeriveInfo info : entry.getValue()) {
+ if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
+ // any
+ // wanted
+ // columns?
+ wantedInfo.add(info);
+ }
+
+ if (wantedInfo.size() > 0)
+ result.put(hostCols, wantedInfo);
+ }
+ return result;
+ }
+
+ public String getResourcePath() {
+ return getCubeDescResourcePath(name);
+ }
+
+ public static String getCubeDescResourcePath(String descName) {
+ return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
+ }
+
+ // ============================================================================
+
+ public HBaseMappingDesc getHBaseMapping() {
+ return hbaseMapping;
+ }
+
+ public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
+ this.hbaseMapping = hbaseMapping;
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public void setConfig(KylinConfig config) {
+ this.config = config;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getModelName() {
+ return modelName;
+ }
+
+ public void setModelName(String modelName) {
+ this.modelName = modelName;
+ }
+
+ public DataModelDesc getModel() {
+ return model;
+ }
+
+ public void setModel(DataModelDesc model) {
+ this.model = model;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getFactTable() {
+ return model.getFactTable();
+ }
+
+ public TableDesc getFactTableDesc() {
+ return model.getFactTableDesc();
+ }
+
+ public String[] getNullStrings() {
+ return nullStrings;
+ }
+
+ public List<DimensionDesc> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(List<DimensionDesc> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public List<MeasureDesc> getMeasures() {
+ return measures;
+ }
+
+ public void setMeasures(List<MeasureDesc> measures) {
+ this.measures = measures;
+ }
+
+ public RowKeyDesc getRowkey() {
+ return rowkey;
+ }
+
+ public void setRowkey(RowKeyDesc rowkey) {
+ this.rowkey = rowkey;
+ }
+
+ public String getSignature() {
+ return signature;
+ }
+
+ public void setSignature(String signature) {
+ this.signature = signature;
+ }
+
+ public List<String> getNotifyList() {
+ return notifyList;
+ }
+
+ public void setNotifyList(List<String> notifyList) {
+ this.notifyList = notifyList;
+ }
+
+ public List<String> getStatusNeedNotify() {
+ return statusNeedNotify;
+ }
+
+ public void setStatusNeedNotify(List<String> statusNeedNotify) {
+ this.statusNeedNotify = statusNeedNotify;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ CubeDesc cubeDesc = (CubeDesc) o;
+
+ if (!name.equals(cubeDesc.name))
+ return false;
+ if (!getFactTable().equals(cubeDesc.getFactTable()))
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 0;
+ result = 31 * result + name.hashCode();
+ result = 31 * result + getFactTable().hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "CubeDesc [name=" + name + "]";
+ }
+
+ public boolean checkSignature() {
+ if (StringUtils.isBlank(getSignature())) {
+ return true;
+ }
+ return calculateSignature().equals(getSignature());
+ }
+
+ public String calculateSignature() {
+ MessageDigest md = null;
+ try {
+ md = MessageDigest.getInstance("MD5");
+ StringBuilder sigString = new StringBuilder();
+ sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
+
+ byte[] signature = md.digest(sigString.toString().getBytes());
+ return new String(Base64.encodeBase64(signature));
+ } catch (NoSuchAlgorithmException | JsonProcessingException e) {
+ throw new RuntimeException("Failed to calculate signature");
+ }
+ }
+
+ public Map<String, TblColRef> buildColumnNameAbbreviation() {
+ Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>();
+ for (TblColRef col : listDimensionColumnsExcludingDerived()) {
+ r.put(col.getName(), col);
+ }
+ return r;
+ }
+
+ public void init(KylinConfig config, Map<String, TableDesc> tables) {
+ this.errors.clear();
+ this.config = config;
+
+ if (this.modelName == null || this.modelName.length() == 0) {
+ this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
+ }
+
+ this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
+
+ if (this.model == null) {
+ this.addError("No data model found with name '" + modelName + "'.");
+ }
+
+ for (DimensionDesc dim : dimensions) {
+ dim.init(this, tables);
+ }
+
+ sortDimAndMeasure();
+ initDimensionColumns();
+ initMeasureColumns();
+
+ rowkey.init(this);
+ if (hbaseMapping != null) {
+ hbaseMapping.init(this);
+ }
+
+ initMeasureReferenceToColumnFamily();
+
+ // check all dimension columns are presented on rowkey
+ List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
+ if (rowkey.getRowKeyColumns().length != dimCols.size()) {
+ addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). ");
+ }
+ }
+
+ private void initDimensionColumns() {
+ for (DimensionDesc dim : dimensions) {
+ JoinDesc join = dim.getJoin();
+
+ // init dimension columns
+ ArrayList<TblColRef> dimCols = Lists.newArrayList();
+ String[] colStrs = dim.getColumn();
+
+ // when column is omitted, special case
+ if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
+ for (TblColRef col : join.getForeignKeyColumns()) {
+ dimCols.add(initDimensionColRef(col));
+ }
+ }
+ // normal case
+ else {
+ if (colStrs == null || colStrs.length == 0)
+ throw new IllegalStateException("Dimension column must not be blank " + dim);
+
+ for (String colStr : colStrs) {
+ dimCols.add(initDimensionColRef(dim, colStr));
+ }
+
+ // fill back column ref in hierarchy
+ if (dim.isHierarchy()) {
+ for (int i = 0; i < dimCols.size(); i++)
+ dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
+ }
+ }
+
+ TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
+ dim.setColumnRefs(dimColArray);
+
+ // init derived columns
+ TblColRef[] hostCols = dimColArray;
+ if (dim.isDerived()) {
+ String[] derived = dim.getDerived();
+ String[][] split = splitDerivedColumnAndExtra(derived);
+ String[] derivedNames = split[0];
+ String[] derivedExtra = split[1];
+ TblColRef[] derivedCols = new TblColRef[derivedNames.length];
+ for (int i = 0; i < derivedNames.length; i++) {
+ derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
+ }
+ initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+ }
+
+ // PK-FK derive the other side
+ if (join != null) {
+ TblColRef[] fk = join.getForeignKeyColumns();
+ TblColRef[] pk = join.getPrimaryKeyColumns();
+
+ allColumns.addAll(Arrays.asList(fk));
+ allColumns.addAll(Arrays.asList(pk));
+ for (int i = 0; i < fk.length; i++) {
+ int find = ArrayUtils.indexOf(hostCols, fk[i]);
+ if (find >= 0) {
+ TblColRef derivedCol = initDimensionColRef(pk[i]);
+ initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
+ }
+ }
+ /** disable this code as we don't need fk be derived from pk
+ for (int i = 0; i < pk.length; i++) {
+ int find = ArrayUtils.indexOf(hostCols, pk[i]);
+ if (find >= 0) {
+ TblColRef derivedCol = initDimensionColRef(fk[i]);
+ initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
+ }
+ }
+ */
+ }
+ }
+ }
+
+ private String[][] splitDerivedColumnAndExtra(String[] derived) {
+ String[] cols = new String[derived.length];
+ String[] extra = new String[derived.length];
+ for (int i = 0; i < derived.length; i++) {
+ String str = derived[i];
+ int cut = str.indexOf(":");
+ if (cut >= 0) {
+ cols[i] = str.substring(0, cut);
+ extra[i] = str.substring(cut + 1).trim();
+ } else {
+ cols[i] = str;
+ extra[i] = "";
+ }
+ }
+ return new String[][] { cols, extra };
+ }
+
+ private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) {
+ initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null);
+ }
+
+ private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
+ if (hostCols.length == 0 || derivedCols.length == 0)
+ throw new IllegalStateException("host/derived columns must not be empty");
+
+ // Although FK derives PK automatically, user unaware of this can declare PK as derived dimension explicitly.
+ // In that case, derivedCols[] will contain a FK which is transformed from the PK by initDimensionColRef().
+ // Must drop FK from derivedCols[] before continue.
+ for (int i = 0; i < derivedCols.length; i++) {
+ if (ArrayUtils.contains(hostCols, derivedCols[i])) {
+ derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
+ extra = (String[]) ArrayUtils.remove(extra, i);
+ i--;
+ }
+ }
+
+ Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
+ List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
+ if (infoList == null) {
+ hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
+ }
+ infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
+
+ for (int i = 0; i < derivedCols.length; i++) {
+ TblColRef derivedCol = derivedCols[i];
+ boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
+ derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
+ }
+ }
+
+ private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
+ TableDesc table = dim.getTableDesc();
+ ColumnDesc col = table.findColumnByName(colName);
+ if (col == null)
+ throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
+
+ TblColRef ref = new TblColRef(col);
+
+ // always use FK instead PK, FK could be shared by more than one lookup tables
+ JoinDesc join = dim.getJoin();
+ if (join != null) {
+ int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), ref);
+ if (idx >= 0) {
+ ref = join.getForeignKeyColumns()[idx];
+ }
+ }
+ return initDimensionColRef(ref);
+ }
+
+ private TblColRef initDimensionColRef(TblColRef ref) {
+ TblColRef existing = findColumnRef(ref.getTable(), ref.getName());
+ if (existing != null) {
+ return existing;
+ }
+
+ allColumns.add(ref);
+ dimensionColumns.add(ref);
+
+ Map<String, TblColRef> cols = columnMap.get(ref.getTable());
+ if (cols == null) {
+ columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>());
+ }
+ cols.put(ref.getName(), ref);
+ return ref;
+ }
+
+ private void initMeasureColumns() {
+ if (measures == null || measures.isEmpty()) {
+ return;
+ }
+
+ TableDesc factTable = getFactTableDesc();
+ for (MeasureDesc m : measures) {
+ m.setName(m.getName().toUpperCase());
+
+ if (m.getDependentMeasureRef() != null) {
+ m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
+ }
+
+ FunctionDesc f = m.getFunction();
+ f.setExpression(f.getExpression().toUpperCase());
+ f.initReturnDataType();
+
+ ParameterDesc p = f.getParameter();
+ p.normalizeColumnValue();
+
+ ArrayList<TblColRef> colRefs = Lists.newArrayList();
+ if (p.isColumnType()) {
+ for (String cName : p.getValue().split("\\s*,\\s*")) {
+ ColumnDesc sourceColumn = factTable.findColumnByName(cName);
+ TblColRef colRef = new TblColRef(sourceColumn);
+ colRefs.add(colRef);
+ allColumns.add(colRef);
+ }
+ }
+
+ // for topN
+ if (StringUtils.isNotEmpty(p.getDisplayColumn())) {
+ ColumnDesc sourceColumn = factTable.findColumnByName(p.getDisplayColumn());
+ TblColRef colRef = new TblColRef(sourceColumn);
+ colRefs.add(colRef);
+ measureDisplayColumns.add(colRef);
+ allColumns.add(colRef);
+ }
+
+ if (colRefs.isEmpty() == false)
+ p.setColRefs(colRefs);
+
+ // verify holistic count distinct as a dependent measure
+ if (m.getFunction().isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
+ throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
+ }
+ }
+ }
+
+ private void initMeasureReferenceToColumnFamily() {
+ if (measures == null || measures.size() == 0)
+ return;
+
+ Map<String, MeasureDesc> measureLookup = new HashMap<String, MeasureDesc>();
+ for (MeasureDesc m : measures)
+ measureLookup.put(m.getName(), m);
+ Map<String, Integer> measureIndexLookup = new HashMap<String, Integer>();
+ for (int i = 0; i < measures.size(); i++)
+ measureIndexLookup.put(measures.get(i).getName(), i);
+
+ for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnDesc c : cf.getColumns()) {
+ String[] colMeasureRefs = c.getMeasureRefs();
+ MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length];
+ int[] measureIndex = new int[colMeasureRefs.length];
+ for (int i = 0; i < colMeasureRefs.length; i++) {
+ measureDescs[i] = measureLookup.get(colMeasureRefs[i]);
+ measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]);
+ }
+ c.setMeasures(measureDescs);
+ c.setMeasureIndex(measureIndex);
+ c.setColumnFamilyName(cf.getName());
+ }
+ }
+ }
+
+ private void sortDimAndMeasure() {
+ sortDimensionsByID();
+ sortMeasuresByID();
+ for (DimensionDesc dim : dimensions) {
+ sortHierarchiesByLevel(dim.getHierarchy());
+ }
+ }
+
+ private void sortDimensionsByID() {
+ Collections.sort(dimensions, new Comparator<DimensionDesc>() {
+ @Override
+ public int compare(DimensionDesc d1, DimensionDesc d2) {
+ Integer id1 = d1.getId();
+ Integer id2 = d2.getId();
+ return id1.compareTo(id2);
+ }
+ });
+ }
+
+ private void sortMeasuresByID() {
+ if (measures == null) {
+ measures = Lists.newArrayList();
+ }
+
+// Collections.sort(measures, new Comparator<MeasureDesc>() {
+// @Override
+// public int compare(MeasureDesc m1, MeasureDesc m2) {
+// Integer id1 = m1.getId();
+// Integer id2 = m2.getId();
+// return id1.compareTo(id2);
+// }
+// });
+ }
+
+ private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
+ if (hierarchies != null) {
+ Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
+ @Override
+ public int compare(HierarchyDesc h1, HierarchyDesc h2) {
+ Integer level1 = Integer.parseInt(h1.getLevel());
+ Integer level2 = Integer.parseInt(h2.getLevel());
+ return level1.compareTo(level2);
+ }
+ });
+ }
+ }
+
+ public boolean hasHolisticCountDistinctMeasures() {
+ for (MeasureDesc measure : measures) {
+ if (measure.getFunction().isHolisticCountDistinct()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public long getRetentionRange() {
+ return retentionRange;
+ }
+
+ public void setRetentionRange(long retentionRange) {
+ this.retentionRange = retentionRange;
+ }
+
+ public long[] getAutoMergeTimeRanges() {
+ return autoMergeTimeRanges;
+ }
+
+ public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
+ this.autoMergeTimeRanges = autoMergeTimeRanges;
+ }
+
+ /**
+ * Add error info and thrown exception out
+ *
+ * @param message
+ */
+ public void addError(String message) {
+ addError(message, false);
+ }
+
+ /**
+ * @param message error message
+ * @param silent if throw exception
+ */
+ public void addError(String message, boolean silent) {
+ if (!silent) {
+ throw new IllegalStateException(message);
+ } else {
+ this.errors.add(message);
+ }
+ }
+
+ public List<String> getError() {
+ return this.errors;
+ }
+
+ public HBaseMappingDesc getHbaseMapping() {
+ return hbaseMapping;
+ }
+
+ public void setHbaseMapping(HBaseMappingDesc hbaseMapping) {
+ this.hbaseMapping = hbaseMapping;
+ }
+
+ public void setNullStrings(String[] nullStrings) {
+ this.nullStrings = nullStrings;
+ }
+
+ public int getStorageType() {
+ return storageType;
+ }
+
+ void setStorageType(int storageType) {
+ this.storageType = storageType;
+ }
+
+ public int getEngineType() {
+ return engineType;
+ }
+
+ void setEngineType(int engineType) {
+ this.engineType = engineType;
+ }
+
+ public List<TblColRef> getAllColumnsNeedDictionary() {
+ List<TblColRef> result = Lists.newArrayList();
+
+ for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) {
+ TblColRef colRef = rowKeyColDesc.getColRef();
+ if (rowkey.isUseDictionary(colRef)) {
+ result.add(colRef);
+ }
+ }
+
+ for (TblColRef colRef : measureDisplayColumns) {
+ if (!result.contains(colRef))
+ result.add(colRef);
+ }
+ return result;
+ }
+
+ public LinkedHashSet<TblColRef> getMeasureDisplayColumns() {
+ return measureDisplayColumns;
+ }
+
+
+ public boolean hasMeasureUsingDictionary() {
+ for (MeasureDesc measureDesc : this.getMeasures()) {
+ if (measureDesc.getFunction().isTopN())
+ return true;
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/DimensionDesc.java
new file mode 100644
index 0000000..3432d08
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/DimensionDesc.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class DimensionDesc {
+
+ @JsonProperty("id")
+ private int id;
+
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("hierarchy")
+ private boolean isHierarchy;
+ @JsonProperty("table")
+ private String table;
+ @JsonProperty("column")
+ private String[] column;
+ @JsonProperty("derived")
+ private String[] derived;
+
+ private TableDesc tableDesc;
+ private JoinDesc join;
+ private HierarchyDesc[] hierarchy;
+
+ // computed
+ private TblColRef[] columnRefs;
+ private TblColRef[] derivedColRefs;
+
+ public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
+ if (name != null)
+ name = name.toUpperCase();
+
+ if (table != null)
+ table = table.toUpperCase();
+
+ tableDesc = tables.get(this.getTable());
+ if (tableDesc == null)
+ throw new IllegalStateException("Can't find table " + table + " for dimension " + name);
+
+ join = null;
+ for (LookupDesc lookup : cubeDesc.getModel().getLookups()) {
+ if (lookup.getTable().equalsIgnoreCase(this.getTable())) {
+ join = lookup.getJoin();
+ break;
+ }
+ }
+
+ if (isHierarchy && this.column.length > 0) {
+ List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
+ for (int i = 0, n = this.column.length; i < n; i++) {
+ String aColumn = this.column[i];
+ HierarchyDesc aHierarchy = new HierarchyDesc();
+ aHierarchy.setLevel(String.valueOf(i + 1));
+ aHierarchy.setColumn(aColumn);
+ hierarchyList.add(aHierarchy);
+ }
+
+ this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
+ }
+
+ if (hierarchy != null && hierarchy.length == 0)
+ hierarchy = null;
+ if (derived != null && derived.length == 0)
+ derived = null;
+
+ if (hierarchy != null) {
+ for (HierarchyDesc h : hierarchy)
+ h.setColumn(h.getColumn().toUpperCase());
+ }
+
+ if (derived != null) {
+ StringUtil.toUpperCaseArray(derived, derived);
+ }
+
+ if (derived != null && join == null) {
+ throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
+ }
+ }
+
+ public boolean isHierarchyColumn(TblColRef col) {
+ if (hierarchy == null)
+ return false;
+
+ for (HierarchyDesc hier : hierarchy) {
+ if (hier.getColumnRef().equals(col))
+ return true;
+ }
+ return false;
+ }
+
+ public boolean isDerived() {
+ return derived != null;
+ }
+
+ public boolean isHierarchy() {
+ return isHierarchy;
+ }
+
+ public void setHierarchy(boolean isHierarchy) {
+ this.isHierarchy = isHierarchy;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public JoinDesc getJoin() {
+ return join;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public TblColRef[] getColumnRefs() {
+ return this.columnRefs;
+ }
+
+ public void setColumnRefs(TblColRef[] colRefs) {
+ this.columnRefs = colRefs;
+ }
+
+ public String[] getColumn() {
+ return this.column;
+ }
+
+ public void setColumn(String[] column) {
+ this.column = column;
+ }
+
+ public HierarchyDesc[] getHierarchy() {
+ return hierarchy;
+ }
+
+ public void setHierarchy(HierarchyDesc[] hierarchy) {
+ this.hierarchy = hierarchy;
+ }
+
+ public String[] getDerived() {
+ return derived;
+ }
+
+ public void setDerived(String[] derived) {
+ this.derived = derived;
+ }
+
+ public TblColRef[] getDerivedColRefs() {
+ return derivedColRefs;
+ }
+
+ public void setDerivedColRefs(TblColRef[] derivedColRefs) {
+ this.derivedColRefs = derivedColRefs;
+ }
+
+ public TableDesc getTableDesc() {
+ return this.tableDesc;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DimensionDesc that = (DimensionDesc) o;
+
+ if (id != that.id)
+ return false;
+ if (!name.equals(that.name))
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id;
+ result = 31 * result + name.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnDesc.java
new file mode 100644
index 0000000..1912e63
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnDesc.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.util.Arrays;
+
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class HBaseColumnDesc {
+
+ @JsonProperty("qualifier")
+ private String qualifier;
+ @JsonProperty("measure_refs")
+ private String[] measureRefs;
+
+ // these two will be assembled at runtime
+ private MeasureDesc[] measures;
+ private int[] measureIndex; // the index on CubeDesc.getMeasures()
+ private String columnFamilyName;
+
+ public String getQualifier() {
+ return qualifier;
+ }
+
+ public void setQualifier(String qualifier) {
+ this.qualifier = qualifier;
+ }
+
+ public String[] getMeasureRefs() {
+ return measureRefs;
+ }
+
+ public void setMeasureRefs(String[] measureRefs) {
+ this.measureRefs = measureRefs;
+ }
+
+ public int[] getMeasureIndex() {
+ return measureIndex;
+ }
+
+ public void setMeasureIndex(int[] index) {
+ this.measureIndex = index;
+ }
+
+ public MeasureDesc[] getMeasures() {
+ return measures;
+ }
+
+ public void setMeasures(MeasureDesc[] measures) {
+ this.measures = measures;
+ }
+
+ public String getColumnFamilyName() {
+ return columnFamilyName;
+ }
+
+ public void setColumnFamilyName(String columnFamilyName) {
+ this.columnFamilyName = columnFamilyName;
+ }
+
+ public int findMeasure(FunctionDesc function) {
+ for (int i = 0; i < measures.length; i++) {
+ if (measures[i].getFunction().equals(function)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public boolean containsMeasure(String refName) {
+ for (String ref : measureRefs) {
+ if (ref.equals(refName))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((columnFamilyName == null) ? 0 : columnFamilyName.hashCode());
+ result = prime * result + ((qualifier == null) ? 0 : qualifier.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HBaseColumnDesc other = (HBaseColumnDesc) obj;
+ if (columnFamilyName == null) {
+ if (other.columnFamilyName != null)
+ return false;
+ } else if (!columnFamilyName.equals(other.columnFamilyName))
+ return false;
+ if (qualifier == null) {
+ if (other.qualifier != null)
+ return false;
+ } else if (!qualifier.equals(other.qualifier))
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "HBaseColumnDesc [qualifier=" + qualifier + ", measureRefs=" + Arrays.toString(measureRefs) + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnFamilyDesc.java
new file mode 100644
index 0000000..73012f3
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseColumnFamilyDesc.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.util.Arrays;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class HBaseColumnFamilyDesc {
+
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("columns")
+ private HBaseColumnDesc[] columns;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public HBaseColumnDesc[] getColumns() {
+ return columns;
+ }
+
+ public void setColumns(HBaseColumnDesc[] columns) {
+ this.columns = columns;
+ }
+
+ @Override
+ public String toString() {
+ return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseMappingDesc.java
new file mode 100644
index 0000000..0a15d5f
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HBaseMappingDesc.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
+
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class HBaseMappingDesc {
+
+ @JsonProperty("column_family")
+ private HBaseColumnFamilyDesc[] columnFamily;
+
+ // point to the cube instance which contain this HBaseMappingDesc instance.
+ private CubeDesc cubeRef;
+
+ public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
+ Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
+ HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
+ if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
+ return result;
+ }
+ for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) {
+ for (HBaseColumnDesc c : cf.getColumns()) {
+ for (MeasureDesc m : c.getMeasures()) {
+ if (m.getFunction().equals(function)) {
+ result.add(c);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ public CubeDesc getCubeRef() {
+ return cubeRef;
+ }
+
+ public void setCubeRef(CubeDesc cubeRef) {
+ this.cubeRef = cubeRef;
+ }
+
+ public HBaseColumnFamilyDesc[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ public void setColumnFamily(HBaseColumnFamilyDesc[] columnFamily) {
+ this.columnFamily = columnFamily;
+ }
+
+ public void init(CubeDesc cubeDesc) {
+ cubeRef = cubeDesc;
+
+ for (HBaseColumnFamilyDesc cf : columnFamily) {
+ cf.setName(cf.getName().toUpperCase());
+
+ for (HBaseColumnDesc c : cf.getColumns()) {
+ c.setQualifier(c.getQualifier().toUpperCase());
+ StringUtil.toUpperCaseArray(c.getMeasureRefs(), c.getMeasureRefs());
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HierarchyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HierarchyDesc.java
new file mode 100644
index 0000000..9dcf05b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/HierarchyDesc.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class HierarchyDesc {
+
+ @JsonProperty("level")
+ private String level;
+ @JsonProperty("column")
+ private String column;
+
+ private TblColRef columnRef;
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+
+ public TblColRef getColumnRef() {
+ return columnRef;
+ }
+
+ public void setColumnRef(TblColRef column) {
+ this.columnRef = column;
+ }
+
+ public String getColumn() {
+ return column;
+ }
+
+ public void setColumn(String columnName) {
+ this.column = columnName;
+ }
+
+ @Override
+ public String toString() {
+ return "HierarchyDesc [level=" + level + ", column=" + column + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyColDesc.java
new file mode 100644
index 0000000..86e4a53
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyColDesc.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author yangli9
+ *
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class RowKeyColDesc {
+
+ @JsonProperty("column")
+ private String column;
+ @JsonProperty("length")
+ private int length;
+ @JsonProperty("dictionary")
+ private String dictionary;
+ @JsonProperty("mandatory")
+ private boolean mandatory = false;
+
+ // computed
+ private int bitIndex;
+ private TblColRef colRef;
+
+ public String getDictionary() {
+ return dictionary;
+ }
+
+ public String getColumn() {
+ return column;
+ }
+
+ void setColumn(String column) {
+ this.column = column;
+ }
+
+ public int getLength() {
+ return length;
+ }
+
+ public boolean isMandatory() {
+ return mandatory;
+ }
+
+ public int getBitIndex() {
+ return bitIndex;
+ }
+
+ void setBitIndex(int index) {
+ this.bitIndex = index;
+ }
+
+ public TblColRef getColRef() {
+ return colRef;
+ }
+
+ void setColRef(TblColRef colRef) {
+ this.colRef = colRef;
+ }
+
+ public void setDictionary(String dictionary) {
+ this.dictionary = dictionary;
+ }
+
+ @Override
+ public String toString() {
+ return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyDesc.java
new file mode 100644
index 0000000..95249ba
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/RowKeyDesc.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model.v2;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class RowKeyDesc {
+
+ public static class HierarchyMask {
+ public long fullMask;
+ public long[] allMasks;
+ }
+
+ public static class AggrGroupMask {
+ public AggrGroupMask(int size) {
+ groupOneBitMasks = new long[size];
+ }
+
+ public long groupMask;
+ public long groupOneBitMasks[];
+ public long uniqueMask;
+ public long leftoverMask;
+ }
+
+ @JsonProperty("rowkey_columns")
+ private RowKeyColDesc[] rowkeyColumns;
+ @JsonProperty("aggregation_groups")
+ private String[][] aggregationGroups;
+
+ // computed content
+ private CubeDesc cubeDesc;
+ private Map<TblColRef, RowKeyColDesc> columnMap;
+
+ private long fullMask;
+ private long mandatoryColumnMask;
+ private AggrGroupMask[] aggrGroupMasks;
+ private long aggrGroupFullMask;
+ private long tailMask;
+
+ private List<HierarchyMask> hierarchyMasks;
+
+ public RowKeyColDesc[] getRowKeyColumns() {
+ return rowkeyColumns;
+ }
+
+ // search a specific row key col
+ public int getRowKeyIndexByColumnName(String columnName) {
+ if (this.rowkeyColumns == null)
+ return -1;
+
+ for (int i = 0; i < this.rowkeyColumns.length; ++i) {
+ RowKeyColDesc desc = this.rowkeyColumns[i];
+ if (desc.getColumn().equalsIgnoreCase(columnName)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public int getNCuboidBuildLevels() {
+ // N aggregation columns requires N levels of cuboid build
+ // - N columns requires N-1 levels build
+ // - zero tail cuboid needs one more additional level
+ Set<String> aggDims = new HashSet<String>();
+ for (String[] aggrGroup : aggregationGroups) {
+ for (String dim : aggrGroup) {
+ aggDims.add(dim);
+ }
+ }
+ return aggDims.size();
+ }
+
+ public String[][] getAggregationGroups() {
+ return aggregationGroups;
+ }
+
+ public CubeDesc getCubeRef() {
+ return cubeDesc;
+ }
+
+ public void setCubeRef(CubeDesc cubeRef) {
+ this.cubeDesc = cubeRef;
+ }
+
+ public long getFullMask() {
+ return fullMask;
+ }
+
+ public long getMandatoryColumnMask() {
+ return mandatoryColumnMask;
+ }
+
+ public long getAggrGroupFullMask() {
+ return aggrGroupFullMask;
+ }
+
+ public AggrGroupMask[] getAggrGroupMasks() {
+ return aggrGroupMasks;
+ }
+
+ public List<HierarchyMask> getHierarchyMasks() {
+ return hierarchyMasks;
+ }
+
+ public long getTailMask() {
+ return tailMask;
+ }
+
+ public int getColumnBitIndex(TblColRef col) {
+ return getColDesc(col).getBitIndex();
+ }
+
+ public int getColumnLength(TblColRef col) {
+ return getColDesc(col).getLength();
+ }
+
+ public String getDictionary(TblColRef col) {
+ return getColDesc(col).getDictionary();
+ }
+
+ private RowKeyColDesc getColDesc(TblColRef col) {
+ RowKeyColDesc desc = columnMap.get(col);
+ if (desc == null)
+ throw new NullPointerException("Column " + col + " does not exist in row key desc");
+ return desc;
+ }
+
+ public boolean isUseDictionary(int index) {
+ String useDictionary = rowkeyColumns[index].getDictionary();
+ return useDictionary(useDictionary);
+ }
+
+ public boolean isUseDictionary(TblColRef col) {
+ String useDictionary = getDictionary(col);
+ return useDictionary(useDictionary);
+ }
+
+ private boolean useDictionary(String useDictionary) {
+ return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary);
+ }
+
+ public void init(CubeDesc cube) {
+ setCubeRef(cube);
+ Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation();
+
+ buildRowKey(colNameAbbr);
+ buildAggregationGroups(colNameAbbr);
+ buildHierarchyMasks();
+ }
+
+ @Override
+ public String toString() {
+ return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]";
+ }
+
+ private void buildRowKey(Map<String, TblColRef> colNameAbbr) {
+ columnMap = new HashMap<TblColRef, RowKeyColDesc>();
+ mandatoryColumnMask = 0;
+
+ for (int i = 0; i < rowkeyColumns.length; i++) {
+ RowKeyColDesc rowKeyColDesc = rowkeyColumns[i];
+ String column = rowKeyColDesc.getColumn();
+ rowKeyColDesc.setColumn(column.toUpperCase());
+ rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1);
+ rowKeyColDesc.setColRef(colNameAbbr.get(column));
+ if (rowKeyColDesc.getColRef() == null) {
+ throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc);
+ }
+
+ columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc);
+
+ if (rowKeyColDesc.isMandatory()) {
+ mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex();
+ }
+ }
+ }
+
+ private void buildAggregationGroups(Map<String, TblColRef> colNameAbbr) {
+ if (aggregationGroups == null) {
+ aggregationGroups = new String[0][];
+ }
+
+ for (int i = 0; i < aggregationGroups.length; i++) {
+ StringUtil.toUpperCaseArray(aggregationGroups[i], this.aggregationGroups[i]);
+ }
+
+ for (int i = 0; i < this.rowkeyColumns.length; i++) {
+ int index = rowkeyColumns[i].getBitIndex();
+ this.fullMask |= 1L << index;
+ }
+
+ this.aggrGroupMasks = new AggrGroupMask[aggregationGroups.length];
+ for (int i = 0; i < this.aggregationGroups.length; i++) {
+ String[] aggGrp = this.aggregationGroups[i];
+ AggrGroupMask mask = new AggrGroupMask(aggGrp.length);
+
+ for (int j = 0; j < aggGrp.length; j++) {
+ TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase());
+ if (aggCol == null) {
+ throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName());
+ }
+ Integer index = getColumnBitIndex(aggCol);
+ mask.groupMask |= 1L << index;
+ mask.groupOneBitMasks[j] = 1L << index;
+ this.aggrGroupFullMask |= 1L << index;
+ }
+ this.aggrGroupMasks[i] = mask;
+ }
+
+ this.tailMask = fullMask ^ mandatoryColumnMask ^ aggrGroupFullMask;
+
+ // unique mask = (bits in this group) - (bits in following groups)
+ // leftover mask = (tail bits) + (bits in following groups) - (bits in
+ // this group)
+ for (int i = 0; i < aggrGroupMasks.length; i++) {
+ AggrGroupMask mask = aggrGroupMasks[i];
+
+ mask.uniqueMask = mask.groupMask;
+ for (int j = i + 1; j < aggrGroupMasks.length; j++) {
+ mask.uniqueMask &= ~aggrGroupMasks[j].groupMask;
+ }
+
+ mask.leftoverMask = tailMask;
+ for (int j = i + 1; j < aggrGroupMasks.length; j++) {
+ mask.leftoverMask |= aggrGroupMasks[j].groupMask;
+ }
+ mask.leftoverMask &= ~mask.groupMask;
+ }
+ }
+
+ private void buildHierarchyMasks() {
+ this.hierarchyMasks = new ArrayList<HierarchyMask>();
+
+ for (DimensionDesc dimension : this.cubeDesc.getDimensions()) {
+ HierarchyDesc[] hierarchies = dimension.getHierarchy();
+ if (hierarchies == null || hierarchies.length == 0)
+ continue;
+
+ HierarchyMask mask = new HierarchyMask();
+ ArrayList<Long> allMaskList = new ArrayList<Long>();
+ for (int i = 0; i < hierarchies.length; i++) {
+ TblColRef hColumn = hierarchies[i].getColumnRef();
+ Integer index = getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+
+ if ((tailMask & bit) > 0)
+ continue; // ignore levels in tail, they don't participate
+ // aggregation group combination anyway
+
+ mask.fullMask |= bit;
+ allMaskList.add(mask.fullMask);
+ }
+
+ mask.allMasks = new long[allMaskList.size()];
+ for (int i = 0; i < allMaskList.size(); i++)
+ mask.allMasks[i] = allMaskList.get(i);
+
+ this.hierarchyMasks.add(mask);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
index 2f8fc7b..7d7710c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
@@ -19,9 +19,8 @@
package org.apache.kylin.cube.model.validation;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
import org.apache.kylin.cube.model.validation.rule.FunctionRule;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
/**
@@ -32,16 +31,14 @@ import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
*/
public class CubeMetadataValidator {
@SuppressWarnings("unchecked")
- private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupSizeRule(), new MandatoryColumnRule(), new RowKeyAttrRule() };
+ private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), new RowKeyAttrRule() };
public ValidateContext validate(CubeDesc cube) {
return validate(cube, false);
}
/**
- * @param cubeDesc
- * @param inject
- * inject error into cube desc
+ * @param inject inject error into cube desc
* @return
*/
public ValidateContext validate(CubeDesc cube, boolean inject) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
index e33ec19..7f1f617 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
@@ -25,9 +25,6 @@ import java.util.List;
/**
* Context. Supply all dependent objects for validator
- *
- * @author jianliu
- *
*/
public class ValidateContext {
private List<Result> results = new ArrayList<ValidateContext.Result>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java
new file mode 100644
index 0000000..f63e5f1
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ * find forbid overlaps in each AggregationGroup
+ * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint
+ */
+public class AggregationGroupOverlapRule implements IValidatorRule<CubeDesc> {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
+ * , org.apache.kylin.metadata.validation.ValidateContext)
+ */
+ @Override
+ public void validate(CubeDesc cube, ValidateContext context) {
+
+ int index = 0;
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+
+ if ((agg.getMandatoryColumnMask() & agg.getHierarchyDimsMask()) != 0) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+ }
+ if ((agg.getMandatoryColumnMask() & agg.getJointDimsMask()) != 0) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims");
+ }
+
+ int jointDimNum = 0;
+ for (Long joint : agg.getJointDims()) {
+ jointDimNum += Long.bitCount(joint);
+ if (jointDimNum < 2) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint");
+ }
+
+ int overlapHierarchies = 0;
+ for (AggregationGroup.HierarchyMask mask : agg.getHierarchyMasks()) {
+ long share = (joint & mask.fullMask);
+ if (share != 0) {
+ overlapHierarchies++;
+ }
+ if (Long.bitCount(share) > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy");
+ }
+ }
+
+ if (overlapHierarchies > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies");
+ }
+ }
+
+ if (jointDimNum != Long.bitCount(agg.getJointDimsMask())) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint");
+ }
+
+ index++;
+ }
+
+ }
+
+}
[3/7] incubator-kylin git commit: whitelist
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index c89cce2..f09ce8a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
@@ -114,7 +113,7 @@ public class CubeTupleConverter {
}
// prepare derived columns and filler
- Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(cuboidDims, null);
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboidDims, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
TblColRef[] hostCols = entry.getKey().data;
for (DeriveInfo deriveInfo : entry.getValue()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index b31a078..d2a18c9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -69,7 +69,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
CubeInstance cube = cubeMgr.getCube(cubeName);
CubeDesc cubeDesc = cube.getDescriptor();
FsPermission permission = new FsPermission((short) 0777);
- for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) {
String cfName = cf.getName();
Path columnFamilyPath = new Path(input, cfName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
index 9f97b0e..509df42 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
@@ -66,7 +66,7 @@ public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWrita
inputMeasures = new Object[cubeDesc.getMeasures().size()];
keyValueCreators = Lists.newArrayList();
- for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index cdc259b..864765e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -47,7 +47,7 @@ public class CubeHTableUtil {
tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
}
- for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName());
cf.setMaxVersions(1);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 31cce7b..c4dc0b5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -76,7 +76,7 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
- for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
index f99ddb5..205c0db 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
@@ -140,7 +140,7 @@ public class HBaseMROutput2 implements IMROutput2 {
List<RowValueDecoder> valueDecoderList = Lists.newArrayList();
List<MeasureDesc> measuresDescs = Lists.newArrayList();
- for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
valueDecoderList.add(new RowValueDecoder(colDesc));
for (MeasureDesc measure : colDesc.getMeasures()) {
@@ -238,7 +238,7 @@ public class HBaseMROutput2 implements IMROutput2 {
@Override
public void doReducerOutput(ByteArrayWritable key, Object[] value, Reducer.Context context) throws IOException, InterruptedException {
if (keyValueCreators.size() == 0) {
- for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
keyValueCreators.add(new KeyValueCreator(seg.getCubeDesc(), colDesc));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index e7c4cf5..8cb8f13 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -274,7 +274,7 @@ public class HBaseMROutput2Transition implements IMROutput2 {
if (rowValueDecoders == null) {
List<RowValueDecoder> valueDecoderList = Lists.newArrayList();
List<MeasureDesc> measuresDescs = Lists.newArrayList();
- for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cfDesc : seg.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
valueDecoderList.add(new RowValueDecoder(colDesc));
for (MeasureDesc measure : colDesc.getMeasures()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index ba79305..68a2e76 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -55,7 +55,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
@Test
public void testDecode() throws Exception {
CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
+ HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];
MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
BigDecimal sum = new BigDecimal("333.1234567");
@@ -86,7 +86,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
+ HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];
MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
[2/7] incubator-kylin git commit: minor: give find command -L option
to follow all symbolic links
Posted by ma...@apache.org.
minor: give find command -L option to follow all symbolic links
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2d5d3ef9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2d5d3ef9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2d5d3ef9
Branch: refs/heads/KYLIN-242
Commit: 2d5d3ef9bf068753128df9282fc00ad3a5e8f4ca
Parents: a63242c
Author: honma <ho...@ebay.com>
Authored: Mon Nov 23 13:36:49 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 23 13:36:49 2015 +0800
----------------------------------------------------------------------
build/bin/cleanup_streaming_files.sh | 2 +-
build/bin/find-hive-dependency.sh | 4 ++--
build/bin/sample.sh | 2 +-
build/bin/streaming_rolllog.sh | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5d3ef9/build/bin/cleanup_streaming_files.sh
----------------------------------------------------------------------
diff --git a/build/bin/cleanup_streaming_files.sh b/build/bin/cleanup_streaming_files.sh
index 3837a2d..4fc9f0e 100644
--- a/build/bin/cleanup_streaming_files.sh
+++ b/build/bin/cleanup_streaming_files.sh
@@ -8,7 +8,7 @@ fi
cd $KYLIN_HOME/logs
-for pidfile in `find . -name "$1_1*"`
+for pidfile in `find -L . -name "$1_1*"`
do
pidfile=`echo "$pidfile" | cut -c 3-`
echo "pidfile:$pidfile"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5d3ef9/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index de14260..fa28321 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -36,7 +36,7 @@ else
hcatalog_home=${HCAT_HOME}
fi
-hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
if [ -z "$hcatalog" ]
then
@@ -45,7 +45,7 @@ then
fi
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
echo "hive dependency: $hive_dependency"
export hive_dependency
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5d3ef9/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 6931457..a71d43f 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -1,7 +1,7 @@
#!/bin/bash
dir=$(dirname ${0})
source ${dir}/check-env.sh
-job_jar=`find ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
+job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
echo "Going to create sample tables in hive..."
cd ${KYLIN_HOME}/sample_cube/data
hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5d3ef9/build/bin/streaming_rolllog.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_rolllog.sh b/build/bin/streaming_rolllog.sh
index 9380f49..1fa5bb7 100644
--- a/build/bin/streaming_rolllog.sh
+++ b/build/bin/streaming_rolllog.sh
@@ -7,6 +7,6 @@ KYLIN_LOG_HOME=${KYLIN_HOME}/logs
cd ${KYLIN_LOG_HOME}
timestamp=`date +%Y_%m_%d_%H_%M_%S`
tarfile=logs_archived_at_${timestamp}.tar
-files=`find . ! -name '*.tar' -type f -mtime +1` # keep two days' log
+files=`find -L . ! -name '*.tar' -type f -mtime +1` # keep two days' log
echo ${files} | xargs tar -cvf ${tarfile}
echo ${files} | xargs rm
\ No newline at end of file
[6/7] incubator-kylin git commit: whitelist
Posted by ma...@apache.org.
whitelist
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2d5f725b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2d5f725b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2d5f725b
Branch: refs/heads/KYLIN-242
Commit: 2d5f725bf3ea47972ed1fd4c2f67791124abba17
Parents: 2d5d3ef
Author: honma <ho...@ebay.com>
Authored: Fri Nov 27 16:47:28 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 30 11:16:53 2015 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 7 +-
.../persistence/RootPersistentEntity.java | 2 +
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 319 +++----
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 78 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 235 +++--
.../gridtable/CuboidToGridTableMapping.java | 10 +-
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 6 +-
.../org/apache/kylin/cube/kv/RowConstants.java | 2 +
.../kylin/cube/model/AggregationGroup.java | 261 ++++++
.../org/apache/kylin/cube/model/CubeDesc.java | 179 ++--
.../apache/kylin/cube/model/DimensionDesc.java | 121 +--
.../kylin/cube/model/HBaseMappingDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 73 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 227 +----
.../org/apache/kylin/cube/model/SelectRule.java | 30 +
.../kylin/cube/model/v2/CubeBuildTypeEnum.java | 39 +
.../apache/kylin/cube/model/v2/CubeDesc.java | 867 +++++++++++++++++++
.../kylin/cube/model/v2/DimensionDesc.java | 239 +++++
.../kylin/cube/model/v2/HBaseColumnDesc.java | 138 +++
.../cube/model/v2/HBaseColumnFamilyDesc.java | 58 ++
.../kylin/cube/model/v2/HBaseMappingDesc.java | 96 ++
.../kylin/cube/model/v2/HierarchyDesc.java | 68 ++
.../kylin/cube/model/v2/RowKeyColDesc.java | 92 ++
.../apache/kylin/cube/model/v2/RowKeyDesc.java | 295 +++++++
.../model/validation/CubeMetadataValidator.java | 9 +-
.../cube/model/validation/ValidateContext.java | 3 -
.../rule/AggregationGroupOverlapRule.java | 85 ++
.../validation/rule/AggregationGroupRule.java | 170 ++++
.../rule/AggregationGroupSizeRule.java | 66 --
.../rule/IKylinValidationConstants.java | 6 +-
.../validation/rule/MandatoryColumnRule.java | 75 --
.../model/validation/rule/RowKeyAttrRule.java | 16 +-
.../cube/upgrade/V2/CubeDescUpgraderV2.java | 290 +++++++
.../cube/upgrade/V2/CubeMetadataUpgradeV2.java | 177 ++++
.../org/apache/kylin/cube/util/CubingUtils.java | 30 +-
.../kylin/cube/AggregationGroupRuleTest.java | 101 +++
.../cube/AggregationGroupSizeRuleTest.java | 101 ---
.../kylin/cube/DictionaryManagerTest.java | 4 +-
.../kylin/cube/MandatoryColumnRuleTest.java | 57 --
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 64 +-
.../kylin/metadata/MetadataUpgradeTest.java | 2 +-
.../apache/kylin/dict/DictionaryManager.java | 10 +-
.../kylin/metadata/model/MeasureDesc.java | 15 -
.../kylin/engine/mr/BatchCubingJobBuilder.java | 2 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 3 +-
.../mr/steps/MapContextGTRecordWriter.java | 2 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../engine/mr/steps/MergeDictionaryStep.java | 2 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 3 +-
.../apache/kylin/engine/spark/SparkCubing.java | 5 +-
.../spark/cube/DefaultTupleConverter.java | 3 +-
.../localmeta/cube_desc/sample.json | 257 ++++++
.../apache/kylin/rest/service/CubeService.java | 4 +-
.../rest/controller/CubeControllerTest.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 6 +-
.../hbase/cube/v2/CubeTupleConverter.java | 3 +-
.../kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-
.../storage/hbase/steps/CubeHFileMapper.java | 2 +-
.../storage/hbase/steps/CubeHTableUtil.java | 2 +-
.../storage/hbase/steps/HBaseCuboidWriter.java | 2 +-
.../storage/hbase/steps/HBaseMROutput2.java | 4 +-
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../hbase/steps/RowValueDecoderTest.java | 4 +-
68 files changed, 3773 insertions(+), 1274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 5a0fee7..c033117 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -297,10 +297,9 @@ public class FactTableGenerator {
JoinDesc jDesc = dim.getJoin();
if (jDesc == null) {
// column on fact table used directly as a dimension
- for (String aColumn : dim.getColumn()) {
- if (!factTableCol2LookupCol.containsKey(aColumn))
- usedCols.add(aColumn);
- }
+ String aColumn = dim.getColumn();
+ if (!factTableCol2LookupCol.containsKey(aColumn))
+ usedCols.add(aColumn);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index bc72c1e..c8177e6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -90,6 +90,8 @@ abstract public class RootPersistentEntity implements AclEntity, Serializable {
public void setLastModified(long lastModified) {
this.lastModified = lastModified;
}
+
+
public void updateRandomUuid() {
setUuid(UUID.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index d753a20..7c6a57f 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -88,6 +87,7 @@ public class BasicTest {
}
}
+
@Test
public void testxx() throws InterruptedException {
byte[][] data = new byte[10000000][];
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4592b15..4338f54 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -167,7 +167,7 @@ public class CubeManager implements IRealizationProvider {
return null;
DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), "true", col, factTableValueProvider);
+ DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col, factTableValueProvider);
if (dictInfo != null) {
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index d7e7d9c..b0774cf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -19,30 +19,38 @@
package org.apache.kylin.cube.cuboid;
import java.util.ArrayList;
-import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
+
public class Cuboid implements Comparable<Cuboid> {
private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
+ //smaller is better
+ public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return ComparisonChain.start().compare(Long.bitCount(o1), Long.bitCount(o2)).compare(o1, o2).result();
+ }
+ };
+
public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
return findById(cube, Bytes.toLong(cuboidID));
}
@@ -64,29 +72,33 @@ public class Cuboid implements Comparable<Cuboid> {
}
public static boolean isValid(CubeDesc cube, long cuboidID) {
- RowKeyDesc rowkey = cube.getRowkey();
-
- if (cuboidID < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+ if (isValid(agg, cuboidID)) {
+ return true;
+ }
}
- if (checkBaseCuboid(rowkey, cuboidID)) {
- return true;
- }
+ return false;
+ }
- if (checkMandatoryColumns(rowkey, cuboidID) == false) {
- return false;
+ private static boolean isValid(AggregationGroup agg, long cuboidID) {
+ if (cuboidID < 0) {
+ throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
}
-
- if (checkAggregationGroup(rowkey, cuboidID) == false) {
- return false;
+ if ((cuboidID & ~agg.getPartialCubeFullMask()) != 0) {
+ return false; //a cuboid's parent within agg is at most partialCubeFullMask
}
+ return checkMandatoryColumns(agg, cuboidID) && checkHierarchy(agg, cuboidID);
+ }
- if (checkHierarchy(rowkey, cuboidID) == false) {
- return false;
+ public static List<AggregationGroup> getValidAggGroupForCuboid(CubeDesc cubeDesc, long cuboidID) {
+ List<AggregationGroup> ret = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ if (isValid(agg, cuboidID)) {
+ ret.add(agg);
+ }
}
-
- return true;
+ return ret;
}
public static long getBaseCuboidId(CubeDesc cube) {
@@ -97,168 +109,96 @@ public class Cuboid implements Comparable<Cuboid> {
return findById(cube, getBaseCuboidId(cube));
}
- private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
- // add mandantory
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
- if (cuboidID < mandatoryColumnMask) {
- cuboidID = cuboidID | mandatoryColumnMask;
- }
-
- // add hierarchy
- for (DimensionDesc dimension : cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- boolean found = false;
- long result = 0;
- if (hierarchies != null && hierarchies.length > 0) {
- for (int i = hierarchies.length - 1; i >= 0; i--) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = rowkey.getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((rowkey.getTailMask() & bit) > 0)
- continue; // ignore levels in tail, they don't participate
-
- if ((bit & cuboidID) > 0) {
- found = true;
- }
-
- if (found == true) {
- result = result | bit;
- }
- }
- cuboidID = cuboidID | result;
+ public static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
+ List<Long> candidates = Lists.newArrayList();
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ Long candidate = translateToValidCuboid(agg, cuboidID);
+ if (candidate != null) {
+ candidates.add(candidate);
}
}
- // find the left-most aggregation group
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
-
- if (leftover == 0) {
- return cuboidID;
- }
-
- if (leftover != 0) {
- cuboidID = cuboidID | mask.leftoverMask;
- return cuboidID;
- }
- }
+ if (candidates.size() == 0) {
+ throw new IllegalStateException("Cannot find parent for :" + cuboidID);
}
- // doesn't have column in aggregation groups
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- if (leftover == 0) {
- // doesn't have column in tail group
- if (cuboidWithoutMandatory != 0) {
- return cuboidID;
- } else {
- // no column (except mandatory), add one column
- long toAddCol = (1 << (BitSet.valueOf(new long[] { rowkey.getTailMask() }).cardinality()));
- // check if the toAddCol belongs to any hierarchy
- List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
- if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
- for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
- long result = toAddCol & hierarchyMasks.fullMask;
- if (result > 0) {
- // replace it with the root col in hierarchy
- toAddCol = hierarchyMasks.allMasks[0];
- break;
- }
- }
- }
- cuboidID = cuboidID | toAddCol;
- return cuboidID;
- }
- }
-
- // has column in tail group
- cuboidID = cuboidID | rowkey.getTailMask();
- return cuboidID;
-
+ return Collections.min(candidates, cuboidSelectComparator);
}
- /** Breadth-First-Search
- * @deprecated due to poor performance
- * @param cube
- * @param cuboidID
- * @return
- */
- private static long translateToValidCuboidDeprecated(CubeDesc cube, long cuboidID) {
- if (Cuboid.isValid(cube, cuboidID)) {
- return cuboidID;
+ private static Long translateToValidCuboid(AggregationGroup agg, long cuboidID) {
+ if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) {
+ //the partial cube might not contain all required dims
+ return null;
}
- HashSet<Long> dedupped = new HashSet<Long>();
- Queue<Long> queue = new LinkedList<Long>();
- List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID);
-
- // check each parent
- addToQueue(queue, parents, dedupped);
- while (queue.size() > 0) {
- long parent = pollFromQueue(queue, dedupped);
- if (Cuboid.isValid(cube, parent)) {
- return parent;
- } else {
- addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped);
+ // add mandantory
+ cuboidID = cuboidID | agg.getMandatoryColumnMask();
+
+ // add hierarchy
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long fullMask = hierarchyMask.fullMask;
+ long intersect = cuboidID & fullMask;
+ if (intersect != 0 && intersect != fullMask) {
+ long lsb = Long.lowestOneBit(intersect);
+ long fillMask = fullMask & ~(lsb - 1);
+ cuboidID |= fillMask;
}
}
- return -1;
- }
-
- private static List<Long> getAllPossibleParents(CubeDesc cube, long cuboidID) {
- List<Long> allPossibleParents = new ArrayList<Long>();
- for (int i = 0; i < cube.getRowkey().getRowKeyColumns().length; i++) {
- long mask = 1L << i;
- long parentId = cuboidID | mask;
- if (parentId != cuboidID) {
- allPossibleParents.add(parentId);
+ // add joint dims
+ for (Long joint : agg.getJointDims()) {
+ if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) {
+ cuboidID = cuboidID | joint;
}
}
- return allPossibleParents;
- }
-
- private static void addToQueue(Queue<Long> queue, List<Long> parents, HashSet<Long> dedupped) {
- Collections.sort(parents);
- for (Long p : parents) {
- if (!dedupped.contains(p)) {
- dedupped.add(p);
- queue.offer(p);
+ if (cuboidID != 0) {
+ return cuboidID;
+ } else {
+ // no column, add one column
+ long nonJointDims = removeBits(agg.getPartialCubeFullMask(), agg.getJointDims());
+ if (nonJointDims != 0) {
+ long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() {
+ @Override
+ public Long apply(HierarchyMask input) {
+ return input.fullMask;
+ }
+ }));
+ if (nonJointNonHierarchy != 0) {
+ //there exists dim that does not belong to any joint or any hierarchy, that's perfect
+ return Long.lowestOneBit(nonJointNonHierarchy);
+ } else {
+ //choose from a hierarchy that does not intersect with any joint dim, only check level 1
+ long allJointDims = agg.getJointDimsMask();
+ int index = 0;
+ for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ long dim = hierarchyMask.allMasks[index];
+ if ((nonJointDims & allJointDims) == 0) {
+ return dim;
+ }
+ }
+ }
}
- }
- }
- private static long pollFromQueue(Queue<Long> queue, HashSet<Long> dedupped) {
- long element = queue.poll();
- dedupped.remove(element);
- return element;
+ return Collections.min(agg.getJointDims(), cuboidSelectComparator);
+ }
}
- private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
- long baseCuboidId = rowkey.getFullMask();
- if (cuboidID > baseCuboidId) {
- throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
+ private static long removeBits(long original, Collection<Long> toRemove) {
+ long ret = original;
+ for (Long joint : toRemove) {
+ ret = ret & ~joint;
}
- return baseCuboidId == cuboidID;
+ return ret;
}
- private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
-
- // note the all-zero cuboid (except for mandatory) is not valid
- if (cuboidID <= mandatoryColumnMask)
- return false;
-
+ private static boolean checkMandatoryColumns(AggregationGroup agg, long cuboidID) {
+ long mandatoryColumnMask = agg.getMandatoryColumnMask();
return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
}
- private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
- List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
+ private static boolean checkHierarchy(AggregationGroup agg, long cuboidID) {
+ List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
// if no hierarchy defined in metadata
if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
return true;
@@ -279,23 +219,9 @@ public class Cuboid implements Comparable<Cuboid> {
return true;
}
- private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
- return leftover == 0 || leftover == mask.leftoverMask;
- }
- }
-
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- return leftover == 0 || leftover == rowkey.getTailMask();
- }
-
// ============================================================================
- private CubeDesc cube;
+ private CubeDesc cubeDesc;
private final long inputID;
private final long id;
private final byte[] idBytes;
@@ -305,8 +231,8 @@ public class Cuboid implements Comparable<Cuboid> {
private volatile CuboidToGridTableMapping cuboidToGridTableMapping = null;
// will translate the cuboidID if it is not valid
- private Cuboid(CubeDesc cube, long originalID, long validID) {
- this.cube = cube;
+ private Cuboid(CubeDesc cubeDesc, long originalID, long validID) {
+ this.cubeDesc = cubeDesc;
this.inputID = originalID;
this.id = validID;
this.idBytes = Bytes.toBytes(id);
@@ -316,7 +242,7 @@ public class Cuboid implements Comparable<Cuboid> {
private List<TblColRef> translateIdToColumns(long cuboidID) {
List<TblColRef> dimesnions = new ArrayList<TblColRef>();
- RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
+ RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
for (int i = 0; i < allColumns.length; i++) {
// NOTE: the order of column in list!!!
long bitmask = 1L << allColumns[i].getBitIndex();
@@ -335,23 +261,30 @@ public class Cuboid implements Comparable<Cuboid> {
// higher level in hierarchy can be ignored when counting aggregation columns
private long eliminateHierarchyAggregation(long id) {
- List<HierarchyMask> hierarchyMaskList = cube.getRowkey().getHierarchyMasks();
- if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
- for (HierarchyMask hierMask : hierarchyMaskList) {
- long[] allMasks = hierMask.allMasks;
- for (int i = allMasks.length - 1; i > 0; i--) {
- long bit = allMasks[i] ^ allMasks[i - 1];
- if ((inputID & bit) != 0) {
- id &= ~allMasks[i - 1];
+ long finalId = id;
+
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ long temp = id;
+ List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks();
+ if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
+ for (HierarchyMask hierMask : hierarchyMaskList) {
+ long[] allMasks = hierMask.allMasks;
+ for (int i = allMasks.length - 1; i > 0; i--) {
+ long bit = allMasks[i] ^ allMasks[i - 1];
+ if ((inputID & bit) != 0) {
+ temp &= ~allMasks[i - 1];
+ if (temp < finalId)
+ finalId = temp;
+ }
}
}
}
}
- return id;
+ return finalId;
}
- public CubeDesc getCube() {
- return cube;
+ public CubeDesc getCubeDesc() {
+ return cubeDesc;
}
public List<TblColRef> getColumns() {
@@ -375,10 +308,6 @@ public class Cuboid implements Comparable<Cuboid> {
return inputID;
}
- public boolean useAncestor() {
- return inputID != id;
- }
-
public boolean requirePostAggregation() {
return requirePostAggregation;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 7a4b00a..777d809 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -26,9 +26,6 @@ import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
/**
* @author yangli9
@@ -41,11 +38,11 @@ public class CuboidCLI {
if ("test".equals(args[0])) {
CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
- simulateCuboidGeneration(cubeDesc);
+ simulateCuboidGeneration(cubeDesc, true);
}
}
- public static int simulateCuboidGeneration(CubeDesc cube) {
+ public static int simulateCuboidGeneration(CubeDesc cube, boolean validate) {
CuboidScheduler scheduler = new CuboidScheduler(cube);
long baseCuboid = Cuboid.getBaseCuboidId(cube);
@@ -64,20 +61,16 @@ public class CuboidCLI {
cuboidQueue.push(sc);
}
}
-
- /** disable this due to poor performance when dimension number is big
- TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
- if (enumCuboids.equals(cuboidSet) == false) {
- throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
- }
- */
- int mathCount = mathCalcCuboidCount(cube);
- if (mathCount != cuboidSet.size()) {
- throw new IllegalStateException("Math cuboid count " + mathCount + ", but actual cuboid count " + cuboidSet.size());
+ if (validate) {
+ //only run this for test purpose, performance is bad when # of dims is large
+ TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
+ if (enumCuboids.equals(cuboidSet) == false) {
+ throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
+ }
}
- return mathCount;
+ return cuboidSet.size();
}
@@ -93,7 +86,7 @@ public class CuboidCLI {
}
public static int[] calculateAllLevelCount(CubeDesc cube) {
- int levels = cube.getRowkey().getNCuboidBuildLevels();
+ int levels = cube.getBuildLevel();
int[] allLevelCounts = new int[levels + 1];
CuboidScheduler scheduler = new CuboidScheduler(cube);
@@ -116,55 +109,4 @@ public class CuboidCLI {
return allLevelCounts;
}
- public static int mathCalcCuboidCount(CubeDesc cube) {
- int result = 1; // 1 for base cuboid
-
- RowKeyDesc rowkey = cube.getRowkey();
- AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
- result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
- }
-
- return result;
- }
-
- private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
- long groupMask = aggrGroupMask.groupMask;
- int n = mathCalcCuboidCount_combination(rowkey, groupMask);
- n -= 2; // exclude group all 1 and all 0
-
- long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
- if (nonUniqueMask > 0) {
- // exclude duplicates caused by non-unique columns
- // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
- n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
- }
-
- if (hasTail) {
- n *= 2; // tail being 1 and 0
- n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
- }
-
- return n;
- }
-
- private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
- if (colMask == 0) // no column selected
- return 0;
-
- int count = 1;
-
- for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
- long hierBits = colMask & hierMask.fullMask;
- if (hierBits != 0) {
- count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
- colMask &= ~hierBits;
- }
- }
-
- count *= Math.pow(2, Long.bitCount(colMask));
-
- return count;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 7596601..93cab06 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -21,180 +21,171 @@ package org.apache.kylin.cube.cuboid;
/**
*/
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class CuboidScheduler {
- private final CubeDesc cubeDef;
- private final int size;
+ private final CubeDesc cubeDesc;
private final long max;
private final Map<Long, List<Long>> cache;
- public CuboidScheduler(CubeDesc cube) {
- this.cubeDef = cube;
- this.size = cube.getRowkey().getRowKeyColumns().length;
+ public CuboidScheduler(CubeDesc cubeDesc) {
+ this.cubeDesc = cubeDesc;
+ int size = this.cubeDesc.getRowkey().getRowKeyColumns().length;
this.max = (long) Math.pow(2, size) - 1;
this.cache = new ConcurrentHashMap<Long, List<Long>>();
}
- public int getCuboidCount() {
- return getCuboidCount(Cuboid.getBaseCuboidId(cubeDef));
- }
-
- private int getCuboidCount(long cuboidId) {
- int r = 1;
- for (Long child : getSpanningCuboid(cuboidId)) {
- r += getCuboidCount(child);
+ public long getParent(long child) {
+ List<Long> candidates = Lists.newArrayList();
+ long baseCuboidID = Cuboid.getBaseCuboidId(cubeDesc);
+ if (child == baseCuboidID) {
+ throw new IllegalStateException();
}
- return r;
- }
-
- public List<Long> getSpanningCuboid(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
- }
-
- List<Long> result = cache.get(cuboid);
- if (result != null) {
- return result;
- }
-
- // smaller sibling's children
- Collection<Long> allPrevOffspring = new HashSet<Long>();
- for (Long sibling : findSmallerSibling(cuboid)) {
- Collection<Long> prevOffsprings = generateChildren(sibling);
- allPrevOffspring.addAll(prevOffsprings);
- }
-
- // my children is my generation excluding smaller sibling's generation
- result = new ArrayList<Long>();
- for (Long offspring : generateChildren(cuboid)) {
- if (!allPrevOffspring.contains(offspring)) {
- result.add(offspring);
+ for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, child)) {
+ boolean thisAggContributed = false;
+ if (agg.getPartialCubeFullMask() == child) {
+ candidates.add(baseCuboidID);
+ continue;
}
- }
- cache.put(cuboid, result);
- return result;
- }
+ //+1 dim
- private Collection<Long> generateChildren(long cuboid) {
- Collection<Long> result = new HashSet<Long>();
+ //add one normal dim (only try the lowest dim)
+ long normalDimsMask = agg.getNormalDimsMask();
+ if (normalDimsMask != 0) {
+ candidates.add(child | Long.lowestOneBit(normalDimsMask));
+ thisAggContributed = true;
+ }
- // generate zero tail cuboid -- the one with all 1 in the first
- // aggregation group and all 0 for the rest bits
- generateZeroTailBase(cuboid, result);
+ for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+ if ((child & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+ if (i == hierarchyMask.allMasks.length - 1) {
+ continue;//match the full hierarchy
+ }
+ if ((agg.getJointDimsMask() & hierarchyMask.dims[i + 1]) == 0) {
+ //only when the hierarchy dim is not among joints
+ candidates.add(child | hierarchyMask.dims[i + 1]);
+ thisAggContributed = true;
+ }
+ break;//if hierarchyMask 111 is matched, won't check 110 or 100
+ }
+ }
+ }
- RowKeyDesc rowkey = cubeDef.getRowkey();
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if (belongTo(cuboidWithoutMandatory, mask) == false)
+ if (thisAggContributed) {
+ //next section is going to append more than 2 dim to child
+ //thisAggContributed means there's already 1 dim added to child
+ //which can safely prune the 2+ dim candidates.
continue;
+ }
- long[] groupOneBitMasks = mask.groupOneBitMasks;
- for (int i = 0; i < groupOneBitMasks.length; i++) {
- long oneBit = groupOneBitMasks[i];
- if ((cuboid & oneBit) == 0)
- continue;
-
- long child = cuboid ^ oneBit;
- if (Cuboid.isValid(cubeDef, child)) {
- result.add(child);
+ //2+ dim candidates
+ for (long joint : agg.getJointDims()) {
+ if ((child & joint) == 0) {
+ candidates.add(child | joint);
}
}
-
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
}
- return result;
+ return Collections.min(candidates, Cuboid.cuboidSelectComparator);
}
- private void generateZeroTailBase(long cuboid, Collection<Long> result) {
- RowKeyDesc rowkey = cubeDef.getRowkey();
-
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
-
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
- long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
- if (zeroTail > 0 && zeroTail != cuboid) {
- result.add(zeroTail);
+ public Set<Long> getPotentialChildren(long parent) {
+ HashSet<Long> set = Sets.newHashSet();
+ if (parent == Cuboid.getBaseCuboidId(cubeDesc)) {
+ //base cuboid is responsible for spawning each agg group's root
+ for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
+ long partialCubeFullMask = agg.getPartialCubeFullMask();
+ if (partialCubeFullMask != parent) {
+ set.add(partialCubeFullMask);
}
}
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
+ return set;
}
- }
- public Collection<Long> findSmallerSibling(long cuboid) {
- if (!Cuboid.isValid(cubeDef, cuboid)) {
- return Collections.emptyList();
+ if (Long.bitCount(parent) == 1) {
+ //do not aggregate apex cuboid
+ return set;
}
- RowKeyDesc rowkey = cubeDef.getRowkey();
+ for (AggregationGroup agg : Cuboid.getValidAggGroupForCuboid(cubeDesc, parent)) {
- // do combination in all related groups
- long groupAllBitMask = 0;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((mask.groupMask & cuboid) > 0) {
- groupAllBitMask |= mask.groupMask;
+ //normal dim section
+ for (long normalDimMask : agg.getNormalDims()) {
+ if ((parent & normalDimMask) != 0) {
+ set.add(parent ^ normalDimMask);
+ }
}
+
+ for (AggregationGroup.HierarchyMask hierarchyMask : agg.getHierarchyMasks()) {
+ for (int i = hierarchyMask.allMasks.length - 1; i >= 0; i--) {
+ if ((parent & hierarchyMask.allMasks[i]) == hierarchyMask.allMasks[i]) {
+ if ((agg.getJointDimsMask() & hierarchyMask.dims[i]) == 0) {
+ //only when the hierarchy dim is not among joints
+ set.add(parent ^ hierarchyMask.dims[i]);
+ }
+ break;//if hierarchyMask 111 is matched, won't check 110 or 100
+ }
+ }
+ }
+
+ //joint dim section
+ for (long joint : agg.getJointDims()) {
+ if ((parent & joint) == joint) {
+ set.add(parent ^ joint);
+ }
+ }
+
}
- long groupBitValue = cuboid & groupAllBitMask;
- long leftBitValue = cuboid & ~groupAllBitMask;
- long[] groupOneBits = bits(groupAllBitMask);
+ return set;
+ }
- Collection<Long> siblings = new HashSet<Long>();
- combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
- return siblings;
+ public int getCuboidCount() {
+ return getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc));
}
- private long[] bits(long groupAllBitMask) {
- int size = Long.bitCount(groupAllBitMask);
- long[] r = new long[size];
- long l = groupAllBitMask;
- int i = 0;
- while (l != 0) {
- long bit = Long.highestOneBit(l);
- r[i++] = bit;
- l ^= bit;
+ private int getCuboidCount(long cuboidId) {
+ int r = 1;
+ for (Long child : getSpanningCuboid(cuboidId)) {
+ r += getCuboidCount(child);
}
return r;
}
- private void combination(long cuboid, Collection<Long> siblings, long[] bitMasks, int offset, long bitValue, int k) {
- if (k == 0) {
- if (Cuboid.isValid(cubeDef, bitValue)) {
- siblings.add(bitValue);
- }
- } else {
- for (int i = offset; i < bitMasks.length; i++) {
- long newBitValue = bitValue | bitMasks[i];
- if (newBitValue < cuboid) {
- combination(cuboid, siblings, bitMasks, i + 1, newBitValue, k - 1);
- }
+ public List<Long> getSpanningCuboid(long cuboid) {
+ if (cuboid > max || cuboid < 0) {
+ throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
+ }
+
+ List<Long> result = cache.get(cuboid);
+ if (result != null) {
+ return result;
+ }
+
+ result = Lists.newArrayList();
+ Set<Long> potentials = getPotentialChildren(cuboid);
+ for (Long potential : potentials) {
+ if (getParent(potential) == cuboid) {
+ result.add(potential);
}
}
- }
- private boolean belongTo(long cuboidWithoutMandatory, AggrGroupMask mask) {
- long groupBits = cuboidWithoutMandatory & mask.groupMask;
- long leftoverBits = cuboidWithoutMandatory & mask.leftoverMask;
- return groupBits > 0 && (leftoverBits == 0 || leftoverBits == mask.leftoverMask);
+ cache.put(cuboid, result);
+ return result;
}
public int getCardinality(long cuboid) {
@@ -206,7 +197,7 @@ public class CuboidScheduler {
}
public List<Long> getAllCuboidIds() {
- final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDef);
+ final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
List<Long> result = Lists.newArrayList();
getSubCuboidIds(baseCuboidId, result);
return result;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index c95e932..428137a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -61,7 +61,7 @@ public class CuboidToGridTableMapping {
// column blocks of metrics
ArrayList<BitSet> metricsColBlocks = Lists.newArrayList();
- for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (int i = 0; i < familyDesc.getColumns().length; i++) {
metricsColBlocks.add(new BitSet());
}
@@ -69,7 +69,7 @@ public class CuboidToGridTableMapping {
// metrics
metrics2gt = LinkedListMultimap.create();
- for (MeasureDesc measure :cuboid.getCube().getMeasures()) {
+ for (MeasureDesc measure :cuboid.getCubeDesc().getMeasures()) {
// Count distinct & holistic count distinct are equals() but different.
// Ensure the holistic version if exists is always the first.
FunctionDesc func = measure.getFunction();
@@ -85,7 +85,7 @@ public class CuboidToGridTableMapping {
// map to column block
int cbIdx = 0;
- for (HBaseColumnFamilyDesc familyDesc : cuboid.getCube().getHbaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
if (hbaseColDesc.containsMeasure(measure.getName())) {
metricsColBlocks.get(cbIdx).set(gtColIdx);
@@ -102,7 +102,7 @@ public class CuboidToGridTableMapping {
}
nMetrics = gtColIdx - nDimensions;
- assert nMetrics == cuboid.getCube().getMeasures().size();
+ assert nMetrics == cuboid.getCubeDesc().getMeasures().size();
}
public int getColumnCount() {
@@ -156,7 +156,7 @@ public class CuboidToGridTableMapping {
public Map<Integer, Integer> getDependentMetricsMap() {
Map<Integer, Integer> result = Maps.newHashMap();
- List<MeasureDesc> measures = cuboid.getCube().getMeasures();
+ List<MeasureDesc> measures = cuboid.getCubeDesc().getMeasures();
for (MeasureDesc child : measures) {
if (child.getDependentMeasureRef() != null) {
boolean ok = false;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index f0ee372..2fbd45a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -54,11 +54,11 @@ public final class InMemCubeBuilderUtils {
}
public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
+ int cardinality = Long.bitCount(cuboidId);
BitSet dimension = new BitSet();
- dimension.set(0, bitSet.cardinality());
+ dimension.set(0, cardinality);
BitSet metrics = new BitSet();
- metrics.set(bitSet.cardinality(), bitSet.cardinality() + measureCount);
+ metrics.set(cardinality, cardinality + measureCount);
return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
index 62dea02..3510915 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.kv;
public class RowConstants {
+ public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
+
// row key fixed length place holder
public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
// row key lower bound
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
new file mode 100644
index 0000000..2e57583
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class AggregationGroup {
+ public static class HierarchyMask {
+ public long fullMask;// 00000111
+ public long[] allMasks;// 00000100,00000110,00000111
+ public long[] dims;// 00000100,00000100,00000001
+ }
+
+ @JsonProperty("includes")
+ private String[] includes;
+ @JsonProperty("select_rule")
+ private SelectRule selectRule;
+
+ //computed
+ private long partialCubeFullMask;
+ private long mandatoryColumnMask;
+ private List<HierarchyMask> hierarchyMasks;
+ private List<Long> jointDims;//each long is a group
+ private long jointDimsMask;
+ private long normalDimsMask;
+ private long hierarchyDimsMask;
+ private List<Long> normalDims;//each long is a single dim
+
+ public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) {
+ Map<String, TblColRef> colNameAbbr = cubeDesc.buildColumnNameAbbreviation();
+
+ buildPartialCubeFullMask(colNameAbbr, rowKeyDesc);
+ buildMandatoryColumnMask(colNameAbbr, rowKeyDesc);
+ buildHierarchyMasks(colNameAbbr, rowKeyDesc);
+ buildJointColumnMask(colNameAbbr, rowKeyDesc);
+ buildJointDimsMask();
+ buildNormalDimsMask();
+ buildHierarchyDimsMask();
+ }
+
+ private void buildPartialCubeFullMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ Preconditions.checkState(this.includes != null);
+ Preconditions.checkState(this.includes.length != 0);
+
+ partialCubeFullMask = 0L;
+ for (String dim : this.includes) {
+ TblColRef hColumn = colNameAbbr.get(dim);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+ partialCubeFullMask |= bit;
+ }
+ }
+
+ private void buildJointColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ jointDims = Lists.newArrayList();
+
+ for (String[] joint_dims : this.selectRule.joint_dims) {
+ if (joint_dims == null || joint_dims.length == 0) {
+ continue;
+ }
+
+ long joint = 0L;
+ for (int i = 0; i < joint_dims.length; i++) {
+ TblColRef hColumn = colNameAbbr.get(joint_dims[i]);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+ joint |= bit;
+ }
+
+ Preconditions.checkState(joint != 0);
+ jointDims.add(joint);
+ }
+ }
+
+ private void buildMandatoryColumnMask(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+ mandatoryColumnMask = 0L;
+
+ String[] mandatory_dims = this.selectRule.mandatory_dims;
+ if (mandatory_dims == null || mandatory_dims.length == 0) {
+ return;
+ }
+
+ for (String dim : mandatory_dims) {
+ TblColRef hColumn = colNameAbbr.get(dim);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ mandatoryColumnMask |= 1 << index;
+ }
+
+ }
+
+ private void buildHierarchyMasks(Map<String, TblColRef> colNameAbbr, RowKeyDesc rowKeyDesc) {
+
+ HierarchyMask mask = new HierarchyMask();
+ this.hierarchyMasks = new ArrayList<HierarchyMask>();
+
+ for (String[] hierarchy_dims : this.selectRule.hierarchy_dims) {
+ if (hierarchy_dims == null || hierarchy_dims.length == 0) {
+ continue;
+ }
+
+ ArrayList<Long> allMaskList = new ArrayList<Long>();
+ ArrayList<Long> dimList = new ArrayList<Long>();
+ for (int i = 0; i < hierarchy_dims.length; i++) {
+ TblColRef hColumn = colNameAbbr.get(hierarchy_dims[i]);
+ Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
+ long bit = 1L << index;
+
+ // if ((tailMask & bit) > 0)
+ // continue; // ignore levels in tail, they don't participate
+ // // aggregation group combination anyway
+
+ mask.fullMask |= bit;
+ allMaskList.add(mask.fullMask);
+ dimList.add(bit);
+ }
+
+ Preconditions.checkState(allMaskList.size() == dimList.size());
+ mask.allMasks = new long[allMaskList.size()];
+ mask.dims = new long[dimList.size()];
+ for (int i = 0; i < allMaskList.size(); i++) {
+ mask.allMasks[i] = allMaskList.get(i);
+ mask.dims[i] = dimList.get(i);
+ }
+
+ this.hierarchyMasks.add(mask);
+
+ }
+
+ }
+
+ private void buildNormalDimsMask() {
+ //no joint, no hierarchy, no mandatory
+ long leftover = partialCubeFullMask & ~mandatoryColumnMask;
+ leftover &= ~this.jointDimsMask;
+ for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+ leftover &= ~hierarchyMask.fullMask;
+ }
+
+ this.normalDimsMask = leftover;
+ this.normalDims = bits(leftover);
+ }
+
+ private void buildHierarchyDimsMask() {
+ long ret = 0;
+ for (HierarchyMask mask : hierarchyMasks) {
+ ret |= mask.fullMask;
+ }
+ this.hierarchyDimsMask = ret;
+ }
+
+ private List<Long> bits(long x) {
+ List<Long> r = Lists.newArrayList();
+ long l = x;
+ while (l != 0) {
+ long bit = Long.lowestOneBit(l);
+ r.add(bit);
+ l ^= bit;
+ }
+ return r;
+ }
+
+ public void buildJointDimsMask() {
+ long ret = 0;
+ for (long x : jointDims) {
+ ret |= x;
+ }
+ this.jointDimsMask = ret;
+ }
+
+ public long getMandatoryColumnMask() {
+ return mandatoryColumnMask;
+ }
+
+ public List<HierarchyMask> getHierarchyMasks() {
+ return hierarchyMasks;
+ }
+
+ public int getBuildLevel() {
+ int ret = 0;
+ ret += getNormalDims().size();
+ long allHierarchyMask = 0L;
+ for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
+ ret += hierarchyMask.allMasks.length;
+ allHierarchyMask |= hierarchyMask.fullMask;
+ }
+ for (Long joint : jointDims) {
+ if ((joint & allHierarchyMask) == 0) {
+ ret += 1;
+ }
+ }
+ return ret - 1;
+ }
+
+ public void setIncludes(String[] includes) {
+ this.includes = includes;
+ }
+
+ public void setSelectRule(SelectRule selectRule) {
+ this.selectRule = selectRule;
+ }
+
+ public List<Long> getJointDims() {
+ return jointDims;
+ }
+
+ public long getJointDimsMask() {
+ return jointDimsMask;
+ }
+
+ public long getNormalDimsMask() {
+ return normalDimsMask;
+ }
+
+ public long getHierarchyDimsMask() {
+ return hierarchyDimsMask;
+ }
+
+ public List<Long> getNormalDims() {
+ return normalDims;
+ }
+
+ public long getPartialCubeFullMask() {
+ return partialCubeFullMask;
+ }
+
+ public String[] getIncludes() {
+ return includes;
+ }
+
+ public SelectRule getSelectRule() {
+ return selectRule;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2250945..2df9635 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -32,6 +31,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -58,6 +59,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -67,7 +70,7 @@ import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeDesc extends RootPersistentEntity {
- public static enum DeriveType {
+ public enum DeriveType {
LOOKUP, PK_FK
}
@@ -110,6 +113,8 @@ public class CubeDesc extends RootPersistentEntity {
private RowKeyDesc rowkey;
@JsonProperty("hbase_mapping")
private HBaseMappingDesc hbaseMapping;
+ @JsonProperty("aggregation_gropus")
+ private List<AggregationGroup> aggregationGroups;
@JsonProperty("signature")
private String signature;
@JsonProperty("notify_list")
@@ -171,16 +176,14 @@ public class CubeDesc extends RootPersistentEntity {
}
/**
- * Find FunctionDesc by Full Expression.
- *
- * @return
+ * @return all functions from each measure.
*/
- public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
+ public List<FunctionDesc> listAllFunctions() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
for (MeasureDesc m : measures) {
- if (m.getFunction().equals(manualFunc))
- return m.getFunction();
+ functions.add(m.getFunction());
}
- return null;
+ return functions;
}
public TblColRef findColumnRef(String table, String column) {
@@ -191,14 +194,6 @@ public class CubeDesc extends RootPersistentEntity {
return cols.get(column);
}
- public DimensionDesc findDimensionByColumn(TblColRef col) {
- for (DimensionDesc dim : dimensions) {
- if (ArrayUtils.contains(dim.getColumnRefs(), col))
- return dim;
- }
- return null;
- }
-
public DimensionDesc findDimensionByTable(String lookupTableName) {
lookupTableName = lookupTableName.toUpperCase();
for (DimensionDesc dim : dimensions)
@@ -207,28 +202,6 @@ public class CubeDesc extends RootPersistentEntity {
return null;
}
- public DimensionDesc findDimensionByName(String dimName) {
- dimName = dimName.toUpperCase();
- for (DimensionDesc dim : dimensions) {
- if (dimName.equals(dim.getName()))
- return dim;
- }
- return null;
- }
-
- /**
- * Get all functions from each measure.
- *
- * @return
- */
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measures) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
public boolean isDerived(TblColRef col) {
return derivedToHostMap.containsKey(col);
}
@@ -270,14 +243,6 @@ public class CubeDesc extends RootPersistentEntity {
// ============================================================================
- public HBaseMappingDesc getHBaseMapping() {
- return hbaseMapping;
- }
-
- public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
public KylinConfig getConfig() {
return config;
}
@@ -354,6 +319,14 @@ public class CubeDesc extends RootPersistentEntity {
this.rowkey = rowkey;
}
+ public List<AggregationGroup> getAggregationGroups() {
+ return aggregationGroups;
+ }
+
+ public void setAggregationGroups(List<AggregationGroup> aggregationGroups) {
+ this.aggregationGroups = aggregationGroups;
+ }
+
public String getSignature() {
return signature;
}
@@ -389,12 +362,23 @@ public class CubeDesc extends RootPersistentEntity {
if (!name.equals(cubeDesc.name))
return false;
+
if (!getFactTable().equals(cubeDesc.getFactTable()))
return false;
return true;
}
+ public int getBuildLevel() {
+ return Collections.max(Collections2.transform(aggregationGroups, new Function<AggregationGroup, Integer>() {
+ @Nullable
+ @Override
+ public Integer apply(AggregationGroup input) {
+ return input.getBuildLevel();
+ }
+ })) + 1;
+ }
+
@Override
public int hashCode() {
int result = 0;
@@ -414,13 +398,19 @@ public class CubeDesc extends RootPersistentEntity {
}
return calculateSignature().equals(getSignature());
}
-
+
public String calculateSignature() {
- MessageDigest md = null;
+ MessageDigest md;
try {
md = MessageDigest.getInstance("MD5");
StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
+ sigString.append(this.name).append("|")//
+ .append(JsonUtil.writeValueAsString(this.modelName)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.dimensions)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.measures)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.rowkey)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.aggregationGroups)).append("|")//
+ .append(JsonUtil.writeValueAsString(this.hbaseMapping));
byte[] signature = md.digest(sigString.toString().getBytes());
return new String(Base64.encodeBase64(signature));
@@ -455,11 +445,14 @@ public class CubeDesc extends RootPersistentEntity {
dim.init(this, tables);
}
- sortDimAndMeasure();
initDimensionColumns();
initMeasureColumns();
rowkey.init(this);
+ for (AggregationGroup agg : this.aggregationGroups) {
+ agg.init(this, rowkey);
+ }
+
if (hbaseMapping != null) {
hbaseMapping.init(this);
}
@@ -479,35 +472,32 @@ public class CubeDesc extends RootPersistentEntity {
// init dimension columns
ArrayList<TblColRef> dimCols = Lists.newArrayList();
- String[] colStrs = dim.getColumn();
+ String colStrs = dim.getColumn();
// when column is omitted, special case
- if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
+ if ((colStrs == null && dim.isDerived()) || ("{FK}".equalsIgnoreCase(colStrs))) {
for (TblColRef col : join.getForeignKeyColumns()) {
dimCols.add(initDimensionColRef(col));
}
}
// normal case
else {
- if (colStrs == null || colStrs.length == 0)
+ if (StringUtils.isEmpty(colStrs))
throw new IllegalStateException("Dimension column must not be blank " + dim);
- for (String colStr : colStrs) {
- dimCols.add(initDimensionColRef(dim, colStr));
- }
+ dimCols.add(initDimensionColRef(dim, colStrs));
- // fill back column ref in hierarchy
- if (dim.isHierarchy()) {
- for (int i = 0; i < dimCols.size(); i++)
- dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
- }
+ // // fill back column ref in hierarchy
+ // if (dim.isHierarchy()) {
+ // for (int i = 0; i < dimCols.size(); i++)
+ // dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
+ // }
}
- TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
+ TblColRef[] dimColArray = dimCols.toArray(new TblColRef[dimCols.size()]);
dim.setColumnRefs(dimColArray);
// init derived columns
- TblColRef[] hostCols = dimColArray;
if (dim.isDerived()) {
String[] derived = dim.getDerived();
String[][] split = splitDerivedColumnAndExtra(derived);
@@ -517,7 +507,7 @@ public class CubeDesc extends RootPersistentEntity {
for (int i = 0; i < derivedNames.length; i++) {
derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
}
- initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+ initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
}
// PK-FK derive the other side
@@ -528,10 +518,10 @@ public class CubeDesc extends RootPersistentEntity {
allColumns.addAll(Arrays.asList(fk));
allColumns.addAll(Arrays.asList(pk));
for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, fk[i]);
+ int find = ArrayUtils.indexOf(dimColArray, fk[i]);
if (find >= 0) {
TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
+ initDerivedMap(dimColArray[find], DeriveType.PK_FK, dim, derivedCol);
}
}
/** disable this code as we don't need fk be derived from pk
@@ -693,7 +683,7 @@ public class CubeDesc extends RootPersistentEntity {
for (int i = 0; i < measures.size(); i++)
measureIndexLookup.put(measures.get(i).getName(), i);
- for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) {
for (HBaseColumnDesc c : cf.getColumns()) {
String[] colMeasureRefs = c.getMeasureRefs();
MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length];
@@ -709,53 +699,6 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- private void sortDimAndMeasure() {
- sortDimensionsByID();
- sortMeasuresByID();
- for (DimensionDesc dim : dimensions) {
- sortHierarchiesByLevel(dim.getHierarchy());
- }
- }
-
- private void sortDimensionsByID() {
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc d1, DimensionDesc d2) {
- Integer id1 = d1.getId();
- Integer id2 = d2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortMeasuresByID() {
- if (measures == null) {
- measures = Lists.newArrayList();
- }
-
- Collections.sort(measures, new Comparator<MeasureDesc>() {
- @Override
- public int compare(MeasureDesc m1, MeasureDesc m2) {
- Integer id1 = m1.getId();
- Integer id2 = m2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
- if (hierarchies != null) {
- Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
- @Override
- public int compare(HierarchyDesc h1, HierarchyDesc h2) {
- Integer level1 = Integer.parseInt(h1.getLevel());
- Integer level2 = Integer.parseInt(h2.getLevel());
- return level1.compareTo(level2);
- }
- });
- }
- }
-
public boolean hasHolisticCountDistinctMeasures() {
for (MeasureDesc measure : measures) {
if (measure.getFunction().isHolisticCountDistinct()) {
@@ -822,7 +765,7 @@ public class CubeDesc extends RootPersistentEntity {
return storageType;
}
- void setStorageType(int storageType) {
+ public void setStorageType(int storageType) {
this.storageType = storageType;
}
@@ -830,7 +773,7 @@ public class CubeDesc extends RootPersistentEntity {
return engineType;
}
- void setEngineType(int engineType) {
+ public void setEngineType(int engineType) {
this.engineType = engineType;
}
@@ -855,7 +798,6 @@ public class CubeDesc extends RootPersistentEntity {
return measureDisplayColumns;
}
-
public boolean hasMeasureUsingDictionary() {
for (MeasureDesc measureDesc : this.getMeasures()) {
if (measureDesc.getFunction().isTopN())
@@ -864,4 +806,5 @@ public class CubeDesc extends RootPersistentEntity {
return false;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index e75a123..bccae58 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -18,11 +18,10 @@
package org.apache.kylin.cube.model;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
@@ -32,34 +31,27 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class DimensionDesc {
- @JsonProperty("id")
- private int id;
-
@JsonProperty("name")
private String name;
-
- @JsonProperty("hierarchy")
- private boolean isHierarchy;
@JsonProperty("table")
private String table;
@JsonProperty("column")
- private String[] column;
+ private String column;
@JsonProperty("derived")
private String[] derived;
private TableDesc tableDesc;
private JoinDesc join;
- private HierarchyDesc[] hierarchy;
// computed
private TblColRef[] columnRefs;
- private TblColRef[] derivedColRefs;
public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
if (name != null)
@@ -80,29 +72,30 @@ public class DimensionDesc {
}
}
- if (isHierarchy && this.column.length > 0) {
- List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
- for (int i = 0, n = this.column.length; i < n; i++) {
- String aColumn = this.column[i];
- HierarchyDesc aHierarchy = new HierarchyDesc();
- aHierarchy.setLevel(String.valueOf(i + 1));
- aHierarchy.setColumn(aColumn);
- hierarchyList.add(aHierarchy);
- }
-
- this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
- }
+ // if (isHierarchy && this.column.length > 0) {
+ // List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
+ // for (int i = 0, n = this.column.length; i < n; i++) {
+ // String aColumn = this.column[i];
+ // HierarchyDesc aHierarchy = new HierarchyDesc();
+ // aHierarchy.setLevel(String.valueOf(i + 1));
+ // aHierarchy.setColumn(aColumn);
+ // hierarchyList.add(aHierarchy);
+ // }
+ //
+ // this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
+ // }
+ //
+ // if (hierarchy != null && hierarchy.length == 0)
+ // hierarchy = null;
+
+ // if (hierarchy != null) {
+ // for (HierarchyDesc h : hierarchy)
+ // h.setColumn(h.getColumn().toUpperCase());
+ // }
- if (hierarchy != null && hierarchy.length == 0)
- hierarchy = null;
if (derived != null && derived.length == 0)
derived = null;
- if (hierarchy != null) {
- for (HierarchyDesc h : hierarchy)
- h.setColumn(h.getColumn().toUpperCase());
- }
-
if (derived != null) {
StringUtil.toUpperCaseArray(derived, derived);
}
@@ -112,29 +105,10 @@ public class DimensionDesc {
}
}
- public boolean isHierarchyColumn(TblColRef col) {
- if (hierarchy == null)
- return false;
-
- for (HierarchyDesc hier : hierarchy) {
- if (hier.getColumnRef().equals(col))
- return true;
- }
- return false;
- }
-
public boolean isDerived() {
return derived != null;
}
- public boolean isHierarchy() {
- return isHierarchy;
- }
-
- public void setHierarchy(boolean isHierarchy) {
- this.isHierarchy = isHierarchy;
- }
-
public String getTable() {
return table;
}
@@ -143,14 +117,6 @@ public class DimensionDesc {
this.table = table;
}
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
public JoinDesc getJoin() {
return join;
}
@@ -171,22 +137,14 @@ public class DimensionDesc {
this.columnRefs = colRefs;
}
- public String[] getColumn() {
+ public String getColumn() {
return this.column;
}
- public void setColumn(String[] column) {
+ public void setColumn(String column) {
this.column = column;
}
- public HierarchyDesc[] getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(HierarchyDesc[] hierarchy) {
- this.hierarchy = hierarchy;
- }
-
public String[] getDerived() {
return derived;
}
@@ -195,45 +153,22 @@ public class DimensionDesc {
this.derived = derived;
}
- public TblColRef[] getDerivedColRefs() {
- return derivedColRefs;
- }
-
- public void setDerivedColRefs(TblColRef[] derivedColRefs) {
- this.derivedColRefs = derivedColRefs;
- }
-
public TableDesc getTableDesc() {
return this.tableDesc;
}
@Override
public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- DimensionDesc that = (DimensionDesc) o;
-
- if (id != that.id)
- return false;
- if (!name.equals(that.name))
- return false;
-
- return true;
+ throw new NotImplementedException();
}
@Override
public int hashCode() {
- int result = id;
- result = 31 * result + name.hashCode();
- return result;
+ throw new NotImplementedException();
}
@Override
public String toString() {
- return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
+ return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index e390bd6..2ef1e17 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -43,7 +43,7 @@ public class HBaseMappingDesc {
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
- HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
+ HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping();
if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 278b59f..7b12cd7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -18,11 +18,15 @@
package org.apache.kylin.cube.model;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
/**
* @author yangli9
@@ -31,37 +35,74 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class RowKeyColDesc {
+ public enum ColEncodingType {
+ DICT, FIXED_LEN
+ }
+
+ public class ColEncoding {
+ public ColEncodingType type;
+ public Object param;
+
+ public ColEncoding(ColEncodingType type, Object param) {
+ this.type = type;
+ this.param = param;
+ }
+ }
+
@JsonProperty("column")
private String column;
- @JsonProperty("length")
- private int length;
- @JsonProperty("dictionary")
- private String dictionary;
- @JsonProperty("mandatory")
- private boolean mandatory = false;
+ @JsonProperty("encoding")
+ private String encoding;
// computed
+ private ColEncoding colEncoding;
private int bitIndex;
private TblColRef colRef;
- public String getDictionary() {
- return dictionary;
+ public void init() {
+
+ //dict or fix length?
+ Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
+ if (this.encoding.equalsIgnoreCase("dict")) {
+ this.colEncoding = new ColEncoding(ColEncodingType.DICT, null);
+ } else if (this.encoding.startsWith("fixed_length")) {
+ int length = RowConstants.ROWKEY_COL_DEFAULT_LENGTH;
+ if (this.encoding.indexOf(":") > 0) {
+ length = Integer.parseInt(this.encoding.substring(this.encoding.indexOf(":") + 1));
+ }
+ this.colEncoding = new ColEncoding(ColEncodingType.FIXED_LEN, length);
+ } else {
+ throw new IllegalArgumentException("Not supported row key col encoding:" + this.encoding);
+ }
+ }
+
+ public String getEncoding() {
+ return encoding;
+ }
+
+ public void setEncoding(String encoding) {
+ this.encoding = encoding;
}
public String getColumn() {
return column;
}
- void setColumn(String column) {
+ public void setColumn(String column) {
this.column = column;
}
- public int getLength() {
- return length;
+ public boolean isUsingDictionary() {
+ return this.colEncoding.type == ColEncodingType.DICT;
+
}
- public boolean isMandatory() {
- return mandatory;
+ public int getLength() {
+ if (this.colEncoding.type == ColEncodingType.FIXED_LEN) {
+ return (Integer) this.colEncoding.param;
+ } else {
+ throw new IllegalStateException("Not knowing the col's length");
+ }
}
public int getBitIndex() {
@@ -80,13 +121,9 @@ public class RowKeyColDesc {
this.colRef = colRef;
}
- public void setDictionary(String dictionary) {
- this.dictionary = dictionary;
- }
-
@Override
public String toString() {
- return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
+ return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
}
}
[7/7] incubator-kylin git commit: remove dup col in multiple agg group
Posted by ma...@apache.org.
remove dup col in multiple agg group
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2dda1909
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2dda1909
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2dda1909
Branch: refs/heads/KYLIN-242
Commit: 2dda190927fd808310b76b78594bffddd4e33fef
Parents: 2d5f725
Author: honma <ho...@ebay.com>
Authored: Mon Nov 30 11:17:14 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 30 11:17:14 2015 +0800
----------------------------------------------------------------------
.../localmeta/cube/test_kylin_cube_topn.json | 10 -
.../cube/test_kylin_cube_topn_left_join.json | 10 -
.../localmeta/cube_desc/sample.json | 257 -------------------
.../cube_desc/test_kylin_cube_topn_desc.json | 148 -----------
.../test_kylin_cube_topn_left_join_desc.json | 149 -----------
.../test_kylin_cube_without_slr_desc.json | 3 +-
...t_kylin_cube_without_slr_left_join_desc.json | 3 +-
7 files changed, 2 insertions(+), 578 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
deleted file mode 100644
index 903fc15..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "33354455-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "test_kylin_cube_topn",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_topn_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
deleted file mode 100644
index 6f57561..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "44454455-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "test_kylin_cube_topn_left_join",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_topn_left_join_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube_desc/sample.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/sample.json b/examples/test_case_data/localmeta/cube_desc/sample.json
deleted file mode 100644
index 9581b70..0000000
--- a/examples/test_case_data/localmeta/cube_desc/sample.json
+++ /dev/null
@@ -1,257 +0,0 @@
-{
- "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_with_slr_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 2,
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": null,
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": null,
- "derived": [
- "SELLER_TYPE_DESC"
- ],
- "hierarchy": false
- },
- {
- "id": 6,
- "name": "SELLER_ID",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "SELLER_ID"
- ],
- "derived": null,
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "seller_id",
- "length": 18,
- "dictionary": null,
- "mandatory": true
- },
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "slr_segment_cd",
- }
- ],
- "aggregation_groups": [
- {
- "include": [
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "cal_dt"
- ],
- "selective_rules": {
- "hierarchy_dims": [
- [
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- ],
- "mandatory_dims": [
- "mandatory"
- ],
- "joint_dims": [
- [
- "leaf_categ_id",
- "cal_dt"
- ]
- ]
- }
- }
- ]
- },
- "last_modified": 1422435345330,
- "model_name": "test_kylin_inner_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
deleted file mode 100644
index fddbb10..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
+++ /dev/null
@@ -1,148 +0,0 @@
-{
- "uuid": "4334a905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_topn_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "displaycolumn": "SELLER_ID"
- },
- "returntype": "topn(100)"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "cal_dt"
- ]
- ]
- },
- "last_modified": 1422435345330,
- "model_name": "test_kylin_inner_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }, {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
deleted file mode 100644
index 6aecaae..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
+++ /dev/null
@@ -1,149 +0,0 @@
-{
- "uuid": "5445a905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_topn_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "displaycolumn": "SELLER_ID"
- },
- "returntype": "topn(100)"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "cal_dt"
- ]
- ]
- },
- "last_modified": 1422435345330,
- "model_name": "test_kylin_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 3f9957b..4c8b7a9 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -238,8 +238,7 @@
"leaf_categ_id",
"meta_categ_name",
"categ_lvl3_name",
- "categ_lvl2_name",
- "lstg_format_name"
+ "categ_lvl2_name"
]
]
},
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2dda1909/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 907e338..fafafda 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -238,8 +238,7 @@
"leaf_categ_id",
"meta_categ_name",
"categ_lvl3_name",
- "categ_lvl2_name",
- "lstg_format_name"
+ "categ_lvl2_name"
]
]
},