You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/30 04:16:25 UTC
[4/7] incubator-kylin git commit: whitelist
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
new file mode 100644
index 0000000..4e7b99e
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ * find forbid overlaps in each AggregationGroup
+ * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint
+ */
+public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
+ * , org.apache.kylin.metadata.validation.ValidateContext)
+ */
+ @Override
+ public void validate(CubeDesc cube, ValidateContext context) {
+ innerValidateMaxSize(cube, context);
+ }
+
+ /**
+ * @param cube
+ * @param context
+ */
+ private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
+ int maxSize = getMaxAgrGroupSize();
+
+ int index = 0;
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+ if (agg.getIncludes() == null) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " include dims not set");
+ continue;
+ }
+
+ if (agg.getSelectRule() == null) {
+ continue;
+ }
+
+ Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getIncludes() != null) {
+ for (String include : agg.getIncludes()) {
+ includeDims.add(include);
+ }
+ }
+
+ Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().mandatory_dims != null) {
+ for (String include : agg.getSelectRule().mandatory_dims) {
+ mandatoryDims.add(include);
+ }
+ }
+
+ Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().hierarchy_dims != null) {
+ for (String[] ss : agg.getSelectRule().hierarchy_dims) {
+ for (String s : ss)
+ hierarchyDims.add(s);
+ }
+ }
+
+ Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (agg.getSelectRule().joint_dims != null) {
+ for (String[] ss : agg.getSelectRule().joint_dims) {
+ for (String s : ss)
+ jointDims.add(s);
+ }
+ }
+
+ if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " Include dims not containing all the used dims");
+ }
+
+ Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ normalDims.addAll(includeDims);
+ normalDims.removeAll(mandatoryDims);
+ normalDims.removeAll(hierarchyDims);
+ normalDims.removeAll(jointDims);
+
+ int normalDimSize = normalDims.size();
+ int hierarchySize = (agg.getSelectRule().hierarchy_dims == null ? 0 : agg.getSelectRule().hierarchy_dims.length);
+ int jointSize = agg.getSelectRule().joint_dims == null ? 0 : agg.getSelectRule().joint_dims.length;
+ if (normalDimSize + hierarchySize + jointSize > maxSize) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions");
+ }
+
+ if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+ }
+ if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims");
+ }
+
+ int jointDimNum = 0;
+ if (agg.getSelectRule().joint_dims != null) {
+ for (String[] joints : agg.getSelectRule().joint_dims) {
+
+ Set<String> oneJoint = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ for (String s : joints) {
+ oneJoint.add(s);
+ }
+
+ if (oneJoint.size() < 2) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint");
+ }
+ jointDimNum += oneJoint.size();
+
+ int overlapHierarchies = 0;
+ if (agg.getSelectRule().hierarchy_dims != null) {
+ for (String[] oneHierarchy : agg.getSelectRule().hierarchy_dims) {
+ Set<String> share = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ share.addAll(CollectionUtils.intersection(oneJoint, Arrays.asList(oneHierarchy)));
+
+ if (!share.isEmpty()) {
+ overlapHierarchies++;
+ }
+ if (share.size() > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy");
+ }
+ }
+
+ if (overlapHierarchies > 1) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies");
+ }
+ }
+ }
+
+ if (jointDimNum != jointDims.size()) {
+ context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint");
+ }
+ }
+
+ index++;
+ }
+ }
+
+ protected int getMaxAgrGroupSize() {
+ String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
+ return Integer.parseInt(size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index e37b9a9..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- *
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- innerValidateMaxSize(cube, context);
- }
-
- /**
- * @param cube
- * @param context
- */
- private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
- int maxSize = getMaxAgrGroupSize();
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- if (group.length >= maxSize) {
- context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
- }
- }
- }
-
- protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
index 52e5b24..80d4676 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
@@ -20,13 +20,9 @@ package org.apache.kylin.cube.model.validation.rule;
import org.apache.kylin.metadata.MetadataConstants;
-/**
- * @author jianliu
- *
- */
public interface IKylinValidationConstants extends MetadataConstants {
- public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
+ public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 12;
public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index 4585900..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- *
- * @author jianliu
- *
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- Set<String> mands = new HashSet<String>();
- RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
- if (cols == null || cols.length == 0) {
- return;
- }
- for (int i = 0; i < cols.length; i++) {
- RowKeyColDesc rowKeyColDesc = cols[i];
- if (rowKeyColDesc.isMandatory()) {
- mands.add(rowKeyColDesc.getColumn());
- }
- }
- if (mands.isEmpty()) {
- return;
- }
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- for (int j = 0; j < group.length; j++) {
- String col = group[j];
- if (mands.contains(col)) {
- context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
- }
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
index e90f6cd..7f4b6e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
@@ -18,7 +18,6 @@
package org.apache.kylin.cube.model.validation.rule;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
@@ -28,9 +27,6 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
/**
* Validate that only one of "length" and "dictionary" appears on rowkey_column
- *
- * @author jianliu
- *
*/
public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
@@ -61,12 +57,12 @@ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
for (int i = 0; i < rcd.length; i++) {
RowKeyColDesc rd = rcd[i];
- if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
- }
- if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
- }
+ // if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
+ // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
+ // }
+ // if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
+ // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
+ // }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
new file mode 100644
index 0000000..772da17
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeDescUpgraderV2.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.cube.model.v2.CubeDesc;
+import org.apache.kylin.cube.model.v2.DimensionDesc;
+import org.apache.kylin.cube.model.v2.HBaseMappingDesc;
+import org.apache.kylin.cube.model.v2.RowKeyColDesc;
+import org.apache.kylin.cube.model.v2.RowKeyDesc;
+
+import com.google.common.collect.Lists;
+
+public class CubeDescUpgraderV2 {
+
+ @SuppressWarnings("unused")
+ private static final Log logger = LogFactory.getLog(CubeDescUpgraderV2.class);
+ private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
+
+ private String resourcePath;
+
+ private List<String[]> oldHierarchies = Lists.newArrayList();
+ private List<String> oldMandatories = Lists.newArrayList();
+ private String[][] oldAggGroup = null;
+ private Set<String> allRowKeyCols = newIgnoreCaseSet(null);
+
+ public CubeDescUpgraderV2(String resourcePath) {
+ this.resourcePath = resourcePath;
+ }
+
+ public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
+ CubeDesc oldModel = loadOldCubeDesc(resourcePath);
+
+ org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
+ copyUnChangedProperties(oldModel, newModel);
+ upgradeDimension(oldModel, newModel);
+ upgradeRowKey(oldModel, newModel);
+ upgradeHBaseMapping(oldModel, newModel);
+ upgradeAggregationGroup(newModel);//must do at last
+
+ return newModel;
+ }
+
+ private CubeDesc loadOldCubeDesc(String path) throws IOException {
+ ResourceStore store = getStore();
+
+ CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer);
+
+ if (StringUtils.isBlank(ndesc.getName())) {
+ throw new IllegalStateException("CubeDesc name must not be blank");
+ }
+
+ return ndesc;
+ }
+
+ private Set<String> newIgnoreCaseSet(Set<String> input) {
+ Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (input != null)
+ ret.addAll(input);
+ return ret;
+ }
+
+ private String[] toArray(Set<String> input) {
+ return input.toArray(new String[input.size()]);
+ }
+
+ private boolean rowKeyColExistsInMultipleAggGroup() {
+ if (oldAggGroup == null)
+ return false;
+
+ int total = 0;
+ Set<String> overall = newIgnoreCaseSet(null);
+ for (String[] group : oldAggGroup) {
+ Set<String> temp = newIgnoreCaseSet(null);
+ for (String entry : group) {
+
+ overall.add(entry);
+ temp.add(entry);
+ }
+ total += temp.size();
+ }
+ return overall.size() != total;
+ }
+
+ private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) {
+
+ List<AggregationGroup> aggs = Lists.newArrayList();
+ if (oldAggGroup == null || oldAggGroup.length == 0 || //
+ ((oldAggGroup.length == 1) && (oldAggGroup[0].length == allRowKeyCols.size()))) {
+ AggregationGroup agg = new AggregationGroup();
+ agg.setIncludes(toArray(allRowKeyCols));
+ agg.setSelectRule(null);
+ aggs.add(agg);
+ } else {
+ if (rowKeyColExistsInMultipleAggGroup()) {
+ throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!");
+ }
+
+ Set<String> visited = newIgnoreCaseSet(null);
+
+ for (String[] group : oldAggGroup) {
+ AggregationGroup agg = new AggregationGroup();
+
+ Set<String> remaining = newIgnoreCaseSet(allRowKeyCols);
+ remaining.removeAll(visited);
+
+ Set<String> joint = newIgnoreCaseSet(remaining);
+ joint.removeAll(oldMandatories);
+
+ Set<String> groupAsSet = newIgnoreCaseSet(null);
+ for (String entry : group) {
+ groupAsSet.add(entry);
+ }
+ visited.addAll(groupAsSet);
+ joint.removeAll(groupAsSet);
+
+ List<String> mandatories = Lists.newArrayList();
+ List<String[]> hierarchies = Lists.newArrayList();
+
+ for (String s : oldMandatories) {
+ mandatories.add(s);
+ }
+
+ for (String[] h : oldHierarchies) {
+ if (groupAsSet.containsAll(Arrays.asList(h))) {
+ hierarchies.add(h);
+ }
+ }
+
+ agg.setIncludes(toArray(remaining));
+
+ SelectRule selectRule = new SelectRule();
+ selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]);
+ if (joint.size() != 0) {
+ selectRule.joint_dims = new String[1][];
+ selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]);
+ } else {
+ selectRule.joint_dims = new String[0][];
+ }
+ selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]);
+ agg.setSelectRule(selectRule);
+
+ aggs.add(agg);
+
+ }
+ }
+ newModel.setAggregationGroups(aggs);
+ }
+
+ private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ List<DimensionDesc> oldDimensions = oldModel.getDimensions();
+ if (oldDimensions == null) {
+ throw new IllegalArgumentException("dimensions is null");
+ }
+ List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList();
+
+ for (DimensionDesc oldDim : oldDimensions) {
+ if (oldDim.isDerived()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn("{FK}");
+ newDim.setDerived(oldDim.getDerived());
+
+ newDimensions.add(newDim);
+ } else {
+ if (oldDim.isHierarchy()) {
+ oldHierarchies.add(oldDim.getColumn());
+ }
+
+ for (String columnStr : oldDim.getColumn()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn(columnStr);
+ newDim.setDerived(null);
+
+ newDimensions.add(newDim);
+ }
+ }
+ }
+
+ newModel.setDimensions(newDimensions);
+ }
+
+ private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ RowKeyDesc oldRowKey = oldModel.getRowkey();
+ if (oldModel == null) {
+ throw new IllegalArgumentException("RowKeyDesc is null");
+ }
+
+ if (oldRowKey.getRowKeyColumns() == null) {
+ throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null");
+ }
+
+ org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc();
+ org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length];
+ int index = 0;
+ for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) {
+ org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc();
+
+ allRowKeyCols.add(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.isMandatory()) {
+ oldMandatories.add(oldRowKeyCol.getColumn());
+ }
+
+ newRowKeyCol.setColumn(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) {
+ newRowKeyCol.setEncoding("dict");
+ } else if (oldRowKeyCol.getLength() > 0) {
+ newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength());
+ } else {
+ throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength());
+ }
+ cols[index++] = newRowKeyCol;
+ }
+ oldAggGroup = oldRowKey.getAggregationGroups();
+
+ newRowKey.setRowkeyColumns(cols);
+ newModel.setRowkey(newRowKey);
+ }
+
+ private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping();
+ try {
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, hbaseMappingDesc);
+ byte[] blob = os.toByteArray();
+ ByteArrayInputStream is = new ByteArrayInputStream(blob);
+ org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class);
+ newModel.setHbaseMapping(newHBaseMappingDesc);
+
+ } catch (IOException e) {
+ throw new RuntimeException("error when copying HBaseMappingDesc");
+ }
+ }
+
+ private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ newModel.setUuid(oldModel.getUuid());
+ newModel.setName(oldModel.getName());
+ newModel.setDescription(oldModel.getDescription());
+ newModel.setMeasures(oldModel.getMeasures());
+ newModel.setNullStrings(oldModel.getNullStrings());
+ newModel.setModelName(oldModel.getModelName());
+ newModel.setNotifyList(oldModel.getNotifyList());
+ newModel.setLastModified(oldModel.getLastModified());
+ newModel.setStorageType(oldModel.getStorageType());
+ newModel.setEngineType(oldModel.getEngineType());
+ }
+
+ protected static ResourceStore getStore() {
+ return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
new file mode 100644
index 0000000..4d3fce9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V2/CubeMetadataUpgradeV2.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V2;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+
+/**
+ * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1,
+ * that upgrades metadata store from v1(prior kylin 0.7) to v2.
+ * the major difference is that we split cube desc to cube desc + model desc
+ *
+ * this CubeMetadataUpgradeV2 upgrades metadata store from v2(prior kylin 2.1) to v3
+ * the major different is a brand new definition of partial cubes to allow users to select
+ * cuboids more flexibly
+ */
+public class CubeMetadataUpgradeV2 {
+
+ private KylinConfig config = null;
+ private ResourceStore store;
+
+ private List<String> updatedResources = Lists.newArrayList();
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV2.class);
+
+ public CubeMetadataUpgradeV2(String newMetadataUrl) {
+ KylinConfig.destoryInstance();
+ System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl);
+ KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl);
+
+ config = KylinConfig.getInstanceFromEnv();
+ store = getStore();
+ }
+
+ public void upgrade() {
+
+ upgradeCubeDesc();
+ verify();
+ }
+
+ public void verify() {
+ MetadataManager.clearCache();
+ MetadataManager.getInstance(config);
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.clearCache();
+ CubeManager.getInstance(config);
+ ProjectManager.clearCache();
+ ProjectManager.getInstance(config);
+ //cleanup();
+ }
+
+ private List<String> listResourceStore(String pathRoot) {
+ List<String> paths = null;
+ try {
+ paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+ }
+
+ return paths;
+ }
+
+ private void upgradeCubeDesc() {
+ logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+
+ List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
+ for (String path : paths) {
+
+ try {
+ CubeDescUpgraderV2 upgrade = new CubeDescUpgraderV2(path);
+ CubeDesc ndesc = upgrade.upgrade();
+ ndesc.setSignature(ndesc.calculateSignature());
+
+ getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+ updatedResources.add(ndesc.getResourcePath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
+ }
+ }
+
+ }
+
+ private ResourceStore getStore() {
+ return ResourceStore.getStore(config);
+ }
+
+ public static void main(String[] args) {
+
+ if (!(args != null && (args.length == 1 || args.length == 2))) {
+ System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
+ return;
+ }
+
+ String exportFolder = args[0];
+ boolean verify = false;
+ if (args.length == 2 && "verify".equals(args[1])) {
+ System.out.println("Only verify the metadata in folder " + exportFolder);
+ verify = true;
+ }
+
+ CubeMetadataUpgradeV2 instance = null;
+ if (verify) {
+ instance = new CubeMetadataUpgradeV2(exportFolder);
+ instance.verify();
+ } else {
+ File oldMetaFolder = new File(exportFolder);
+ if (!oldMetaFolder.exists()) {
+ System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
+ return;
+ }
+
+ if (!oldMetaFolder.isDirectory()) {
+ System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
+ return;
+ }
+
+ String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v3";//upgrades metadata store to v3 format
+ try {
+ FileUtils.deleteDirectory(new File(newMetadataUrl));
+ FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ instance = new CubeMetadataUpgradeV2(newMetadataUrl);
+ instance.upgrade();
+ logger.info("=================================================================");
+ logger.info("Run CubeMetadataUpgrade completed;");
+
+ }
+
+ logger.info("=================================================================");
+ if (instance.errorMsgs.size() > 0) {
+ logger.info("Here are the error/warning messages, you may need check:");
+ for (String s : instance.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The migration is success.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 0cfd020..ada31b6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -34,7 +34,6 @@
package org.apache.kylin.cube.util;
import java.io.IOException;
-import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -42,14 +41,6 @@ import java.util.Map;
import javax.annotation.Nullable;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
@@ -59,12 +50,25 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
/**
*/
public class CubingUtils {
@@ -82,8 +86,7 @@ public class CubingUtils {
@Nullable
@Override
public Integer[] apply(@Nullable Long cuboidId) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] result = new Integer[bitSet.cardinality()];
+ Integer[] result = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
@@ -100,8 +103,7 @@ public class CubingUtils {
final Map<Long, HyperLogLogPlusCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
result.put(cuboidId, new HyperLogLogPlusCounter(14));
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+ Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
new file mode 100644
index 0000000..6cfaaa4
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author jianliu
+ *
+ */
+public class AggregationGroupRuleTest {
+
+ private CubeDesc cube;
+ private ValidateContext vContext = new ValidateContext();
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
+ this.cube = desc2;
+
+ }
+
+ @Test
+ public void testOneMandatoryColumn() {
+ IValidatorRule<CubeDesc> rule = new AggregationGroupRule() {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+ * #getMaxAgrGroupSize()
+ */
+ @Override
+ protected int getMaxAgrGroupSize() {
+ return 3;
+ }
+ };
+ rule.validate(cube, vContext);
+ vContext.print(System.out);
+ assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
+ assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
+ assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[2].getMessage()
+ // .startsWith("Hierachy column"));
+ }
+
+ @Test
+ public void testAggColumnSize() {
+ AggregationGroupRule rule = new AggregationGroupRule() {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
+ * #getMaxAgrGroupSize()
+ */
+ @Override
+ protected int getMaxAgrGroupSize() {
+ return 20;
+ }
+ };
+ rule.validate(cube, vContext);
+ vContext.print(System.out);
+ assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[0].getMessage()
+ // .startsWith("Aggregation group"));
+ // assertTrue("Failed to validate aggragation group error",
+ // vContext.getResults()[0].getMessage()
+ // .startsWith("Hierachy column"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index e7914c7..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 3;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[2].getMessage()
- // .startsWith("Hierachy column"));
- }
-
- @Test
- public void testAggColumnSize() {
- AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 20;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Aggregation group"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Hierachy column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index d7feb56..0de5c22 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -56,10 +56,10 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
- DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+ DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
System.out.println(JsonUtil.writeValueAsIndentString(info1));
- DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
+ DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col, null);
System.out.println(JsonUtil.writeValueAsIndentString(info2));
assertTrue(info1.getUuid() == info2.getUuid());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
deleted file mode 100644
index 29b7183..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class MandatoryColumnRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
- rule.validate(cube, vContext);
- assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 83e8d94..b45a005 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -83,34 +83,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
}
- @Test
- public void testFindSmallerSibling1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(255);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(133);
- assertEquals("[131]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(127);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(134);
- assertEquals("[131, 133]", sortToString(siblings));
-
- siblings = scheduler.findSmallerSibling(130);
- assertEquals("[129]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(5);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(135);
- assertEquals("[]", siblings.toString());
- }
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
for (long cuboidId : cuboidIds) {
@@ -123,31 +95,6 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
}
- @Test
- public void testFindSmallerSibling2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(511);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110111111"));
- assertEquals("[383]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("101110111"));
- assertEquals("[319]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110000000"));
- assertEquals("[288, 320]", sortToString(siblings));
- }
@Test
public void testGetSpanningCuboid2() {
@@ -234,19 +181,19 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Test
public void testCuboidGeneration1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
public void testCuboidGeneration2() {
CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
public void testCuboidGeneration3() {
CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidCLI.simulateCuboidGeneration(cube);
+ CuboidCLI.simulateCuboidGeneration(cube, true);
}
@Test
@@ -266,6 +213,11 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
}
+ @Test
+ public void testPotentialChild() {
+ assertEquals(1, 2);
+ }
+
private String sortToString(Collection<Long> longs) {
ArrayList<Long> copy = new ArrayList<Long>(longs);
Collections.sort(copy);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
index e37aabc..774ad09 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
@@ -100,7 +100,7 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
Assert.assertTrue(dims.size() > 0);
for (DimensionDesc dim : dims) {
- Assert.assertTrue(dim.getColumn().length > 0);
+ Assert.assertTrue(dim.getColumn() != null);
}
Assert.assertTrue(cubedesc1.getMeasures().size() > 0);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..2a1304a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -223,11 +223,11 @@ public class DictionaryManager {
}
}
- public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, boolean usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
logger.info("building dictionary for " + col);
- TblColRef srcCol = decideSourceData(model, dict, col);
+ TblColRef srcCol = decideSourceData(model, usingDict, col);
String srcTable = srcCol.getTable();
String srcColName = srcCol.getName();
int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
@@ -259,9 +259,9 @@ public class DictionaryManager {
/**
* Decide a dictionary's source data, leverage PK-FK relationship.
*/
- public TblColRef decideSourceData(DataModelDesc model, String dict, TblColRef col) throws IOException {
+ public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, TblColRef col) throws IOException {
// Note FK on fact table is supported by scan the related PK on lookup table
- if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) {
+ if (usingDict) {
// FK on fact table and join type is inner, use PK from lookup instead
if (model.isFactTable(col.getTable())) {
TblColRef pkCol = model.findPKByFK(col, "inner");
@@ -270,7 +270,7 @@ public class DictionaryManager {
}
return col;
} else
- throw new IllegalArgumentException("Unknown dictionary value: " + dict);
+ throw new IllegalArgumentException("Not using Dictionary ");
}
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 1561b1f..de471c1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -28,8 +28,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class MeasureDesc {
- @JsonProperty("id")
- private int id;
@JsonProperty("name")
private String name;
@JsonProperty("function")
@@ -37,14 +35,6 @@ public class MeasureDesc {
@JsonProperty("dependent_measure_ref")
private String dependentMeasureRef;
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
public String getName() {
return name;
}
@@ -91,11 +81,6 @@ public class MeasureDesc {
}
@Override
- public int hashCode() {
- return id;
- }
-
- @Override
public String toString() {
return "MeasureDesc [name=" + name + ", function=" + function + "]";
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 12cac94..3c10c09 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -63,7 +63,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 3: Build Cube
RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
- final int groupRowkeyColumnsCount = rowKeyDesc.getNCuboidBuildLevels();
+ final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
// base cuboid step
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index b97c88a..b656da2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -62,7 +62,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
for (int i = 0; i < dictionaryColumns.size(); i++) {
TblColRef col = dictionaryColumns.get(i);
- String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+ String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
if (cubeDesc.getModel().isFactTable(scanTable)) {
factDictCols.add(i);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index e43d5d1..0c5b9b3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -89,8 +89,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) {
allCuboids.add(cuboidId);
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] indice = new Integer[bitSet.cardinality()];
+ Integer[] indice = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index 6098381..8416d95 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -88,7 +88,7 @@ public class MapContextGTRecordWriter implements ICuboidWriter {
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
keyBuf = rowKeyEncoder.createBuf();
- dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+ dimensions = Long.bitCount(cuboidId);
measureColumnsIndex = new int[measureCount];
for (int i = 0; i < measureCount; i++) {
measureColumnsIndex[i] = dimensions + i;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 286ff02..53c23b8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -99,7 +99,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 68d1481..ea1fbb0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
else {
ret = cubeDesc.getRowkey().isUseDictionary(col);
if (ret) {
- String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col).getTable();
+ String dictTable = DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().isUseDictionary(col), col).getTable();
ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
}
dictsNeedMerging.put(col, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index b73fda4..8fced61 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -112,7 +112,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
CubeDesc cubeDesc = cube.getDescriptor();
for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
- String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), "true", col).getTable();
+ String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(),true, col).getTable();
if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
colsNeedMeringDict.add(col);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fc016fc..410cec7 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -77,8 +77,7 @@ public class CubeSamplingTest {
}
private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- Integer[] indice = new Integer[bitSet.cardinality()];
+ Integer[] indice = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 01d97fd..32d4d2a 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -229,8 +229,7 @@ public class SparkCubing extends AbstractApplication {
final ByteArray[] row_hashcodes = new ByteArray[nRowKey];
for (Long cuboidId : allCuboidIds) {
- BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
- Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
+ Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
int position = 0;
@@ -480,7 +479,7 @@ public class SparkCubing extends AbstractApplication {
final Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fs = FileSystem.get(hbaseConf);
FsPermission permission = new FsPermission((short) 0777);
- for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnFamilyDesc cf : cubeDesc.getHbaseMapping().getColumnFamily()) {
String cfName = cf.getName();
Path columnFamilyPath = new Path(hfileLocation, cfName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 6b3a82c..9c1d681 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -18,7 +18,6 @@
package org.apache.kylin.engine.spark.cube;
import java.nio.ByteBuffer;
-import java.util.BitSet;
import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
@@ -70,7 +69,7 @@ public final class DefaultTupleConverter implements TupleConverter {
Cuboid cuboid = Cuboid.findById(segment.getCubeDesc(), cuboidId);
RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid);
- final int dimensions = BitSet.valueOf(new long[] { cuboidId }).cardinality();
+ final int dimensions = Long.bitCount(cuboidId);
int[] measureColumnsIndex = getMeasureColumnsIndex();
for (int i = 0; i < measureCount; i++) {
measureColumnsIndex[i] = dimensions + i;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/examples/test_case_data/localmeta/cube_desc/sample.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/sample.json b/examples/test_case_data/localmeta/cube_desc/sample.json
new file mode 100644
index 0000000..9581b70
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/sample.json
@@ -0,0 +1,257 @@
+{
+ "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+ "name": "test_kylin_cube_with_slr_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": null,
+ "derived": [
+ "WEEK_BEG_DT"
+ ],
+ },
+ {
+ "id": 1,
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 2,
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ],
+ "derived": null,
+ "hierarchy": true
+ },
+ {
+ "id": 3,
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": [
+ "LSTG_FORMAT_NAME"
+ ],
+ "derived": null,
+ "hierarchy": false
+ },
+ {
+ "id": 4,
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": null,
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 5,
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": null,
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ],
+ "hierarchy": false
+ },
+ {
+ "id": 6,
+ "name": "SELLER_ID",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": [
+ "SELLER_ID"
+ ],
+ "derived": null,
+ "hierarchy": false
+ }
+ ],
+ "measures": [
+ {
+ "id": 1,
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 2,
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 3,
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 4,
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1"
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 5,
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT"
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "seller_id",
+ "length": 18,
+ "dictionary": null,
+ "mandatory": true
+ },
+ {
+ "column": "cal_dt",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "leaf_categ_id",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "meta_categ_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "categ_lvl2_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "categ_lvl3_name",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "lstg_format_name",
+ "length": 12,
+ "dictionary": null,
+ "mandatory": false
+ },
+ {
+ "column": "lstg_site_id",
+ "length": 0,
+ "dictionary": "true",
+ "mandatory": false
+ },
+ {
+ "column": "slr_segment_cd",
+ }
+ ],
+ "aggregation_groups": [
+ {
+ "include": [
+ "leaf_categ_id",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "cal_dt"
+ ],
+ "selective_rules": {
+ "hierarchy_dims": [
+ [
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ ],
+ "mandatory_dims": [
+ "mandatory"
+ ],
+ "joint_dims": [
+ [
+ "leaf_categ_id",
+ "cal_dt"
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "last_modified": 1422435345330,
+ "model_name": "test_kylin_inner_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "notify_list": null,
+ "engine_type": 2,
+ "storage_type": 2
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 6670de1..28b0b29 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -194,7 +194,7 @@ public class CubeService extends BasicService {
}
try {
- int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc);
+ int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc,false);
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
} catch (Exception e) {
getCubeDescManager().removeCubeDesc(createdDesc);
@@ -260,7 +260,7 @@ public class CubeService extends BasicService {
}
CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
- int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
+ int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc,false);
logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");
ProjectManager projectManager = getProjectManager();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index c313cc3..be6e22c 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -89,7 +89,7 @@ public class CubeControllerTest extends ServiceTestBase {
newCube.setModelName(cube.getModelName());
newCube.setModel(cube.getModel());
newCube.setDimensions(cube.getDimensions());
- newCube.setHBaseMapping(cube.getHBaseMapping());
+ newCube.setHbaseMapping(cube.getHbaseMapping());
newCube.setMeasures(cube.getMeasures());
newCube.setConfig(cube.getConfig());
newCube.setRowkey(cube.getRowkey());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index f84e4e6..fcfc335 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -154,7 +154,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<HBaseKeyRange> scans = buildScanRanges(flattenToOrAndFilter(filterD), dimensionsD);
// check involved measures, build value decoder for each each family:column
- List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context);
+ List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHbaseMapping(), metrics, context);
//memory hungry distinct count are pushed down to coprocessor, no need to set threshold any more
//setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2d5f725b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 8813901..6a15bc3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -12,7 +12,6 @@ import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -21,12 +20,9 @@ import org.apache.kylin.cube.kv.RowKeyDecoder;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -104,7 +100,7 @@ public class CubeTupleConverter {
}
// prepare derived columns and filler
- Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(dimCols, null);
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
TblColRef[] hostCols = entry.getKey().data;
for (DeriveInfo deriveInfo : entry.getValue()) {