You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/09 02:31:24 UTC
[11/39] kylin git commit: KYLIN-2423 Model should always include
PK/FK as dimensions
KYLIN-2423 Model should always include PK/FK as dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/855301dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/855301dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/855301dc
Branch: refs/heads/master-hbase0.98
Commit: 855301dc7fc83ece4f03dd981f3184bb46a97105
Parents: 56c7aa5
Author: Li Yang <li...@apache.org>
Authored: Sat Feb 4 15:19:31 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Feb 4 16:12:42 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 76 ++++++++++++++++++++
.../kylin/metadata/model/JoinTableDesc.java | 7 +-
.../apache/kylin/metadata/model/JoinsTree.java | 11 +--
.../metadata/model/ModelDimensionDesc.java | 2 +
.../model_desc/ut_large_dimension_number.json | 2 +-
5 files changed, 90 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index cc599a4..86eea55 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -20,15 +20,18 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -294,6 +297,11 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
+
+ boolean reinit = validate();
+ if (reinit) { // model slightly changed by validate() and must init() again
+ init(config, tables);
+ }
}
private void initJoinTablesForUpgrade() {
@@ -443,6 +451,74 @@ public class DataModelDesc extends RootPersistentEntity {
joinsTree = new JoinsTree(rootFactTableRef, joins);
}
+ private boolean validate() {
+ Set<TblColRef> mcols = new HashSet<>();
+ for (String m : metrics) {
+ mcols.add(findColumn(m));
+ }
+
+ // validate no dup between dimensions/metrics
+ for (ModelDimensionDesc dim : dimensions) {
+ String table = dim.getTable();
+ for (String c : dim.getColumns()) {
+ TblColRef dcol = findColumn(table, c);
+ if (mcols.contains(dcol))
+ throw new IllegalStateException(dcol + " cannot be both dimension and metrics at the same time in " + this);
+ }
+ }
+
+ // validate PK/FK are in dimensions
+ boolean pkfkDimAmended = false;
+ for (Chain chain : joinsTree.tableChains.values()) {
+ pkfkDimAmended = validatePkFkDim(chain.join, mcols) || pkfkDimAmended;
+ }
+ return pkfkDimAmended;
+ }
+
+ private boolean validatePkFkDim(JoinDesc join, Set<TblColRef> mcols) {
+ if (join == null)
+ return false;
+
+ boolean pkfkDimAmended = false;
+
+ for (TblColRef c : join.getForeignKeyColumns()) {
+ if (!mcols.contains(c)) {
+ pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+ }
+ }
+ for (TblColRef c : join.getPrimaryKeyColumns()) {
+ if (!mcols.contains(c)) {
+ pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+ }
+ }
+ return pkfkDimAmended;
+ }
+
+ private boolean validatePkFkDim(TblColRef c) {
+ String t = c.getTableAlias();
+ ModelDimensionDesc dimDesc = null;
+ for (ModelDimensionDesc dim : dimensions) {
+ if (dim.getTable().equals(t)) {
+ dimDesc = dim;
+ break;
+ }
+ }
+
+ if (dimDesc == null) {
+ dimDesc = new ModelDimensionDesc();
+ dimDesc.setTable(t);
+ dimDesc.setColumns(new String[0]);
+ }
+
+ if (ArrayUtils.contains(dimDesc.getColumns(), c.getName()) == false) {
+ String[] newCols = ArrayUtils.add(dimDesc.getColumns(), c.getName());
+ dimDesc.setColumns(newCols);
+ return true;
+ }
+
+ return false;
+ }
+
/**
* Add error info and thrown exception out
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 51e5787..9ca806e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -18,17 +18,18 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+
import org.apache.kylin.metadata.model.DataModelDesc.TableKind;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.io.Serializable;
-
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class JoinTableDesc implements Serializable {
+ private static final long serialVersionUID = 1L;
@JsonProperty("table")
private String table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index c132d0e..3ab9a46 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.model;
-import com.google.common.base.Preconditions;
-
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
@@ -27,8 +25,11 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-public class JoinsTree implements Serializable {
+import com.google.common.base.Preconditions;
+public class JoinsTree implements Serializable {
+ private static final long serialVersionUID = 1L;
+
final Map<String, Chain> tableChains = new LinkedHashMap<>();
public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
@@ -112,7 +113,9 @@ public class JoinsTree implements Serializable {
return chain.join;
}
- static class Chain implements java.io.Serializable {
+ static class Chain implements Serializable {
+ private static final long serialVersionUID = 1L;
+
TableRef table; // pk side
JoinDesc join;
Chain fkSide;
http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index d14a56b..c0ddbad 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -30,6 +30,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ModelDimensionDesc implements Serializable {
+ private static final long serialVersionUID = 1L;
+
@JsonProperty("table")
private String table;
@JsonProperty("columns")
http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json b/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json
index 38b070e..9eeb774 100644
--- a/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json
+++ b/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json
@@ -11,7 +11,7 @@
"table" : "DEFAULT.WIDE_TABLE",
"columns" : ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AH", "AI", "AJ", "AK", "AL", "AM", "AN", "AO", "AP", "AQ", "AR", "AS", "AT", "AU", "AV", "AW", "AX", "AY", "AZ"]
}],
- "metrics" : [ "A" ],
+ "metrics" : [ ],
"filter_condition" : "",
"partition_desc" : {
"partition_date_column" : "DEFAULT.WIDE_TABLE.A",