You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/04 07:20:25 UTC
kylin git commit: KYLIN-2423 Model should always include PK/FK as
dimensions
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2423 [created] 0f772b352
KYLIN-2423 Model should always include PK/FK as dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f772b35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f772b35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f772b35
Branch: refs/heads/KYLIN-2423
Commit: 0f772b3529ef4855b5ea972eaf0f3d82a971dbd9
Parents: 7611338
Author: Li Yang <li...@apache.org>
Authored: Sat Feb 4 15:19:31 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Feb 4 15:19:31 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 76 ++++++++++++++++++++
.../kylin/metadata/model/JoinTableDesc.java | 7 +-
.../apache/kylin/metadata/model/JoinsTree.java | 11 +--
.../metadata/model/ModelDimensionDesc.java | 2 +
4 files changed, 89 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f772b35/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index cc599a4..86eea55 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -20,15 +20,18 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -294,6 +297,11 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
+
+ boolean reinit = validate();
+ if (reinit) { // model slightly changed by validate() and must init() again
+ init(config, tables);
+ }
}
private void initJoinTablesForUpgrade() {
@@ -443,6 +451,74 @@ public class DataModelDesc extends RootPersistentEntity {
joinsTree = new JoinsTree(rootFactTableRef, joins);
}
+ private boolean validate() {
+ Set<TblColRef> mcols = new HashSet<>();
+ for (String m : metrics) {
+ mcols.add(findColumn(m));
+ }
+
+ // validate no dup between dimensions/metrics
+ for (ModelDimensionDesc dim : dimensions) {
+ String table = dim.getTable();
+ for (String c : dim.getColumns()) {
+ TblColRef dcol = findColumn(table, c);
+ if (mcols.contains(dcol))
+ throw new IllegalStateException(dcol + " cannot be both dimension and metrics at the same time in " + this);
+ }
+ }
+
+ // validate PK/FK are in dimensions
+ boolean pkfkDimAmended = false;
+ for (Chain chain : joinsTree.tableChains.values()) {
+ pkfkDimAmended = validatePkFkDim(chain.join, mcols) || pkfkDimAmended;
+ }
+ return pkfkDimAmended;
+ }
+
+ private boolean validatePkFkDim(JoinDesc join, Set<TblColRef> mcols) {
+ if (join == null)
+ return false;
+
+ boolean pkfkDimAmended = false;
+
+ for (TblColRef c : join.getForeignKeyColumns()) {
+ if (!mcols.contains(c)) {
+ pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+ }
+ }
+ for (TblColRef c : join.getPrimaryKeyColumns()) {
+ if (!mcols.contains(c)) {
+ pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+ }
+ }
+ return pkfkDimAmended;
+ }
+
+ private boolean validatePkFkDim(TblColRef c) {
+ String t = c.getTableAlias();
+ ModelDimensionDesc dimDesc = null;
+ for (ModelDimensionDesc dim : dimensions) {
+ if (dim.getTable().equals(t)) {
+ dimDesc = dim;
+ break;
+ }
+ }
+
+ if (dimDesc == null) {
+ dimDesc = new ModelDimensionDesc();
+ dimDesc.setTable(t);
+ dimDesc.setColumns(new String[0]);
+ }
+
+ if (ArrayUtils.contains(dimDesc.getColumns(), c.getName()) == false) {
+ String[] newCols = ArrayUtils.add(dimDesc.getColumns(), c.getName());
+ dimDesc.setColumns(newCols);
+ return true;
+ }
+
+ return false;
+ }
+
/**
* Add error info and thrown exception out
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f772b35/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 51e5787..9ca806e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -18,17 +18,18 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+
import org.apache.kylin.metadata.model.DataModelDesc.TableKind;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.io.Serializable;
-
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class JoinTableDesc implements Serializable {
+ private static final long serialVersionUID = 1L;
@JsonProperty("table")
private String table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f772b35/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index c132d0e..3ab9a46 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.model;
-import com.google.common.base.Preconditions;
-
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
@@ -27,8 +25,11 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-public class JoinsTree implements Serializable {
+import com.google.common.base.Preconditions;
+public class JoinsTree implements Serializable {
+ private static final long serialVersionUID = 1L;
+
final Map<String, Chain> tableChains = new LinkedHashMap<>();
public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
@@ -112,7 +113,9 @@ public class JoinsTree implements Serializable {
return chain.join;
}
- static class Chain implements java.io.Serializable {
+ static class Chain implements Serializable {
+ private static final long serialVersionUID = 1L;
+
TableRef table; // pk side
JoinDesc join;
Chain fkSide;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f772b35/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index d14a56b..c0ddbad 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -30,6 +30,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ModelDimensionDesc implements Serializable {
+ private static final long serialVersionUID = 1L;
+
@JsonProperty("table")
private String table;
@JsonProperty("columns")