You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/28 23:27:04 UTC
[5/5] kylin git commit: KYLIN-1875 Add JoinsTree,
refactor ModelChooser
KYLIN-1875 Add JoinsTree, refactor ModelChooser
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bda8787d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bda8787d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bda8787d
Branch: refs/heads/master
Commit: bda8787d1a5c9bbe7d94798091763e1004a1eb61
Parents: a1a2a4a
Author: Yang Li <li...@apache.org>
Authored: Mon Nov 28 07:49:11 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Nov 29 07:25:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryManager.java | 6 +-
.../kylin/metadata/model/DataModelDesc.java | 30 +-
.../apache/kylin/metadata/model/JoinDesc.java | 11 +
.../apache/kylin/metadata/model/JoinsTree.java | 130 ++++++
.../apache/kylin/metadata/model/TblColRef.java | 4 +
.../kylin/metadata/MetadataManagerTest.java | 11 +-
.../kylin/metadata/model/JoinsTreeTest.java | 74 ++++
.../test_kylin_snowflake_sales_cube.json | 268 ++++++++++++
.../test_kylin_snowflake_model_desc.json | 161 --------
.../test_kylin_snowflake_sales_model.json | 87 ++++
.../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 ++
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 +++++++++++++++++++
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 +++++++
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 ++
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 +++
.../apache/kylin/query/relnode/OLAPContext.java | 2 +
.../kylin/query/relnode/OLAPTableScan.java | 7 +-
.../kylin/query/routing/ModelChooser.java | 55 +--
18 files changed, 1293 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 820299c..6178234 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -339,7 +339,7 @@ public class DictionaryManager {
// find a lookup table that the col joins as FK
for (TableRef lookup : model.getLookupTables()) {
- JoinDesc lookupJoin = model.getPKSideJoinMap().get(lookup);
+ JoinDesc lookupJoin = model.getJoinByPKSide(lookup);
int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), col);
if (find < 0)
continue;
@@ -357,8 +357,8 @@ public class DictionaryManager {
if (join.isInnerJoin() == false)
return false;
- TableRef table = join.getForeignKeyColumns()[0].getTableRef();
- join = model.getPKSideJoinMap().get(table);
+ TableRef table = join.getFKSide();
+ join = model.getJoinByPKSide(table);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index ff92def..d917571 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -97,8 +97,7 @@ public class DataModelDesc extends RootPersistentEntity {
private Set<TableRef> allTableRefs = Sets.newLinkedHashSet();
private Map<String, TableRef> aliasMap = Maps.newHashMap(); // alias => TableRef, a table has exactly one alias
private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // name => TableRef, a table maybe referenced by multiple names
- private Map<TableRef, JoinDesc> pkSideJoinMap = Maps.newHashMap(); // table (PK side) => JoinTable
- private Map<String, List<JoinDesc>> fkSideJoinMap = Maps.newHashMap(); // table (FK side) => JoinDesc
+ private JoinsTree joinsTree;
/**
* Error messages during resolving json metadata
@@ -151,12 +150,12 @@ public class DataModelDesc extends RootPersistentEntity {
return joinTables;
}
- public Map<TableRef, JoinDesc> getPKSideJoinMap() {
- return pkSideJoinMap;
+ public JoinDesc getJoinByPKSide(TableRef table) {
+ return joinsTree.getJoinByPKSide(table);
}
-
- public Map<String, List<JoinDesc>> getFKSideJoinMap() {
- return fkSideJoinMap;
+
+ public JoinsTree getJoinsTree() {
+ return joinsTree;
}
@Deprecated
@@ -282,6 +281,7 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinTablesForUpgrade();
initTableAlias(tables);
initJoinColumns();
+ initJoinsTree();
ModelDimensionDesc.capicalizeStrings(dimensions);
initPartitionDesc();
}
@@ -360,8 +360,6 @@ public class DataModelDesc extends RootPersistentEntity {
}
private void initJoinColumns() {
- pkSideJoinMap.clear();
- fkSideJoinMap.clear();
for (JoinTableDesc joinTable : joinTables) {
TableRef dimTable = joinTable.getTableRef();
@@ -413,15 +411,15 @@ public class DataModelDesc extends RootPersistentEntity {
logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
}
}
+ }
+ }
- // pk/fk side join maps
- pkSideJoinMap.put(dimTable, join);
- List<JoinDesc> list = fkSideJoinMap.get(fkTable.getTableIdentity());
- if (list == null) {
- fkSideJoinMap.put(fkTable.getTableIdentity(), list = Lists.newArrayListWithCapacity(4));
- }
- list.add(join);
+ private void initJoinsTree() {
+ List<JoinDesc> joins = new ArrayList<>();
+ for (JoinTableDesc joinTable : joinTables) {
+ joins.add(joinTable.getJoin());
}
+ joinsTree = new JoinsTree(rootFactTableRef, joins);
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index d3c0745..6489244 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -110,6 +110,14 @@ public class JoinDesc {
Preconditions.checkState(tableRef == cols[i].getTableRef());
}
+ public TableRef getPKSide() {
+ return primaryKeyColumns[0].getTableRef();
+ }
+
+ public TableRef getFKSide() {
+ return foreignKeyColumns[0].getTableRef();
+ }
+
public void sortByFK() {
Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length == primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length);
boolean cont = true;
@@ -174,6 +182,9 @@ public class JoinDesc {
// equals() without alias
public boolean matches(JoinDesc other) {
+ if (other == null)
+ return false;
+
if (!this.type.equalsIgnoreCase(other.getType()))
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
new file mode 100644
index 0000000..a0b267d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+public class JoinsTree {
+
+ final Map<String, Chain> tableChains = new LinkedHashMap<>();
+
+ public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
+ for (JoinDesc join : joins) {
+ for (TblColRef col : join.getForeignKeyColumns())
+ Preconditions.checkState(col.isQualified());
+ for (TblColRef col : join.getPrimaryKeyColumns())
+ Preconditions.checkState(col.isQualified());
+ }
+
+ tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null));
+
+ for (JoinDesc join : joins) {
+ TableRef pkSide = join.getPKSide();
+ Chain fkSide = tableChains.get(join.getFKSide().getAlias());
+ tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide));
+ }
+ }
+
+ public Map<String, String> matches(JoinsTree another) {
+ return matches(another, Collections.<String, String> emptyMap());
+ }
+
+ public Map<String, String> matches(JoinsTree another, Map<String, String> constraints) {
+ Map<String, String> matchUp = new HashMap<>();
+
+ for (Chain chain : tableChains.values()) {
+ if (matchInTree(chain, another, constraints, matchUp) == false)
+ return null;
+ }
+
+ return matchUp;
+ }
+
+ private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) {
+ String thisAlias = chain.table.getAlias();
+ if (matchUp.containsKey(thisAlias))
+ return true;
+
+ String constraint = constraints.get(thisAlias);
+ if (constraint != null) {
+ return matchChain(chain, another.tableChains.get(constraint), matchUp);
+ }
+
+ for (Chain anotherChain : another.tableChains.values()) {
+ if (matchChain(chain, anotherChain, matchUp)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean matchChain(Chain chain, Chain anotherChain, Map<String, String> matchUp) {
+ String thisAlias = chain.table.getAlias();
+ String anotherAlias = anotherChain.table.getAlias();
+
+ String curMatch = matchUp.get(thisAlias);
+ if (curMatch != null)
+ return curMatch.equals(anotherAlias);
+ if (curMatch == null && matchUp.values().contains(anotherAlias))
+ return false;
+
+ boolean matches = false;
+ if (chain.join == null) {
+ matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc());
+ } else {
+ matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp);
+ }
+
+ if (matches) {
+ matchUp.put(thisAlias, anotherAlias);
+ }
+ return matches;
+ }
+
+ public JoinDesc getJoinByPKSide(TableRef table) {
+ Chain chain = tableChains.get(table.getAlias());
+ if (chain == null)
+ return null;
+ else
+ return chain.join;
+ }
+
+ static class Chain {
+ TableRef table; // pk side
+ JoinDesc join;
+ Chain fkSide;
+
+ public Chain(TableRef table, JoinDesc join, Chain fkSide) {
+ this.table = table;
+ this.join = join;
+ this.fkSide = fkSide;
+ if (join != null) {
+ Preconditions.checkArgument(table == join.getPKSide());
+ Preconditions.checkArgument(fkSide.table == join.getFKSide());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 2cfbafc..bf8d36b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -123,6 +123,10 @@ public class TblColRef implements Serializable {
return table;
}
+ public boolean isQualified() {
+ return table != null;
+ }
+
public String getTableAlias() {
return table != null ? table.getAlias() : "UNKNOWN_ALIAS";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index fcae8d4..204b4e0 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -76,17 +76,20 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testSnowflakeDataModel() throws Exception {
- DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_model_desc");
+ DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model");
Assert.assertTrue(model.getDimensions().size() > 0);
try {
- model.findTable("TEST_KYLIN_COUNTRY");
+ model.findTable("KYLIN_COUNTRY");
Assert.fail();
} catch (IllegalArgumentException ex) {
// excepted
}
- Assert.assertNotNull(model.findColumn("BUYER_COUNTRY"));
- Assert.assertNotNull(model.findColumn("SELLER_COUNTRY"));
+
+ Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+ Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+ Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+ Assert.assertNotNull(model.findColumn("BUYER_ID"));
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
new file mode 100644
index 0000000..f52ef45
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class JoinsTreeTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBasics() {
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ JoinsTree joinsTree = model.getJoinsTree();
+
+ Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY");
+ assertTrue(chain.table == model.findTable("BUYER_COUNTRY"));
+ assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT"));
+ assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES"));
+ assertTrue(chain.fkSide.fkSide.join == null);
+ assertTrue(chain.fkSide.fkSide.fkSide == null);
+ }
+
+ @Test
+ public void testMatch() {
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ JoinsTree joinsTree = model.getJoinsTree();
+
+ Map<String, String> matchUp = joinsTree.matches(joinsTree);
+ for (Entry<String, String> e : matchUp.entrySet()) {
+ assertTrue(e.getKey().equals(e.getValue()));
+ }
+ assertTrue(model.getAllTables().size() == matchUp.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
new file mode 100644
index 0000000..ee0d68c
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
@@ -0,0 +1,268 @@
+{
+ "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "last_modified" : 1451468470824,
+ "name" : "test_kylin_snowflake_sales_cube",
+ "model_name" : "test_kylin_snowflake_sales_model",
+ "description" : null,
+ "null_string" : null,
+ "dimensions" : [ {
+ "name" : "TRANS_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "TRANS_ID",
+ "derived" : null
+ }, {
+ "name" : "CAL_DT",
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+ }, {
+ "name" : "META_CATEG_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEG_LVL2_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEG_LVL3_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "SELLER_ID",
+ "derived" : null
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "BUYER_ID",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "OPS_USER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "OPS_USER_ID",
+ "derived" : null
+ }, {
+ "name" : "OPS_REGION",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "OPS_REGION",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "SELLER_CNT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
+ },
+ "returntype" : "topn(100)"
+ }
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "BUYER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "SELLER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "TRANS_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "PART_DT",
+ "encoding" : "date"
+ }, {
+ "column" : "LEAF_CATEG_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "META_CATEG_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL2_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL3_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_SITE_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "OPS_USER_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "OPS_REGION",
+ "encoding" : "dict"
+ } ]
+ },
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+ } ]
+ }, {
+ "name" : "F2",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "mandatory_dims" : [ "PART_DT" ],
+ "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
+ [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
+ }
+ }, {
+ "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "partition_date_start" : 1325376000000,
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
deleted file mode 100644
index b06a217..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
+++ /dev/null
@@ -1,161 +0,0 @@
-{
- "uuid": "ac0f4ee2-1dcb-4b07-a38e-4c298563e0e3",
- "name": "test_kylin_snowflake_model_desc",
- "lookups": [
- {
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join": {
- "type": "left",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SITES",
- "join": {
- "type": "left",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "join": {
- "type": "left",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_KYLIN_COUNTRY",
- "alias": "BUYER_COUNTRY",
- "join": {
- "type": "left",
- "primary_key": [
- "COUNTRY"
- ],
- "foreign_key": [
- "BUYER_COUNTRY"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_KYLIN_COUNTRY",
- "alias": "SELLER_COUNTRY",
- "join": {
- "type": "left",
- "primary_key": [
- "COUNTRY"
- ],
- "foreign_key": [
- "SELLER_COUNTRY"
- ]
- }
- }
- ],
- "dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD",
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID",
- "BUYER_COUNTRY",
- "SELLER_COUNTRY"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- },
- {
- "table": "BUYER_COUNTRY",
- "columns": [
- "country",
- "name"
- ]
- },
- {
- "table": "SELLER_COUNTRY",
- "columns": [
- "country",
- "name"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID"
- ],
- "last_modified": 1422435345352,
- "fact_table": "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition": null,
- "partition_desc": {
- "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start": 0,
- "partition_type": "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
new file mode 100644
index 0000000..84c6979
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
@@ -0,0 +1,87 @@
+{
+ "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
+ "name" : "test_kylin_snowflake_sales_model",
+ "lookups" : [ {
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "CAL_DT" ],
+ "foreign_key" : [ "PART_DT" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
+ "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_ACCOUNT",
+ "alias" : "BUYER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "BUYER_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_ACCOUNT",
+ "alias" : "SELLER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "SELLER_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_COUNTRY",
+ "alias" : "BUYER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_COUNTRY",
+ "alias" : "SELLER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
+ }],
+ "dimensions" : [ {
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
+ }, {
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "columns" : ["CAL_DT", "WEEK_BEG_DT"]
+ }, {
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
+ }, {
+ "table" : "BUYER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "SELLER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "BUYER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
+ }, {
+ "table" : "SELLER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
+ } ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT",
+ "SELLER_ID",
+ "LSTG_FORMAT_NAME"
+ ],
+ "last_modified" : 1422435345362,
+ "fact_table" : "SNOWTEST.KYLIN_SALES",
+ "filter_condition" : null,
+ "partition_desc" : {
+ "partition_date_column" : "SNOWTEST.KYLIN_SALES.PART_DT",
+ "partition_date_start" : 1325376000000,
+ "partition_type" : "APPEND"
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
new file mode 100644
index 0000000..8d39cbc
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
@@ -0,0 +1,28 @@
+{
+ "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
+
+ "name" : "KYLIN_ACCOUNT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "ACCOUNT_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "3",
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "ACCOUNT_COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "ACCOUNT_CONTACT",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
new file mode 100644
index 0000000..655d4e7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -0,0 +1,408 @@
+{
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+
+ "name" : "KYLIN_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "KYLIN_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "KYLIN_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "KYLIN_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "KYLIN_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
new file mode 100644
index 0000000..abee6d1
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
@@ -0,0 +1,152 @@
+{
+ "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+
+ "name" : "KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LEAF_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "CATEG_BUSN_MGR",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "CATEG_BUSN_UNIT",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "REGN_CATEG",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "USER_DEFINED_FIELD1",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "USER_DEFINED_FIELD3",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "KYLIN_GROUPINGS_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
+ "name" : "KYLIN_GROUPINGS_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "KYLIN_GROUPINGS_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "KYLIN_GROUPINGS_UPD_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "13",
+ "name" : "META_CATEG_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "14",
+ "name" : "META_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "15",
+ "name" : "CATEG_LVL2_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "16",
+ "name" : "CATEG_LVL3_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "17",
+ "name" : "CATEG_LVL4_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "18",
+ "name" : "CATEG_LVL5_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "19",
+ "name" : "CATEG_LVL6_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "20",
+ "name" : "CATEG_LVL7_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "21",
+ "name" : "CATEG_LVL2_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "22",
+ "name" : "CATEG_LVL3_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "23",
+ "name" : "CATEG_LVL4_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "24",
+ "name" : "CATEG_LVL5_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "25",
+ "name" : "CATEG_LVL6_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "26",
+ "name" : "CATEG_LVL7_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "27",
+ "name" : "CATEG_FLAGS",
+ "datatype" : "decimal"
+ }, {
+ "id" : "28",
+ "name" : "ADULT_CATEG_YN",
+ "datatype" : "string"
+ }, {
+ "id" : "29",
+ "name" : "DOMAIN_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "30",
+ "name" : "USER_DEFINED_FIELD5",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "VCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "32",
+ "name" : "GCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "33",
+ "name" : "MOVE_TO",
+ "datatype" : "decimal"
+ }, {
+ "id" : "34",
+ "name" : "SAP_CATEGORY_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "35",
+ "name" : "SRC_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "36",
+ "name" : "BSNS_VRTCL_NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
new file mode 100644
index 0000000..64708c7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
@@ -0,0 +1,24 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+
+ "name" : "KYLIN_COUNTRY",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "2",
+ "name" : "LATITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "3",
+ "name" : "LONGITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "4",
+ "name" : "NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
new file mode 100644
index 0000000..4aa82b5
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
@@ -0,0 +1,56 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+
+ "name" : "KYLIN_SALES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TRANS_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "PART_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "LSTG_FORMAT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "4",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "5",
+ "name" : "LSTG_SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "6",
+ "name" : "SLR_SEGMENT_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "PRICE",
+ "datatype" : "decimal(19,4)"
+ }, {
+ "id" : "8",
+ "name" : "ITEM_COUNT",
+ "datatype" : "bigint"
+ }, {
+ "id" : "9",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "10",
+ "name" : "BUYER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "11",
+ "name" : "OPS_USER_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "OPS_REGION",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 615d802..5fb2c5c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -33,6 +33,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -127,6 +128,7 @@ public class OLAPContext {
public Set<TblColRef> filterColumns = new HashSet<>();
public TupleFilter filter;
public List<JoinDesc> joins = new LinkedList<>();
+ public JoinsTree joinsTree;
private List<TblColRef> sortColumns;
private List<SQLDigest.OrderEnum> sortOrders;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 3c2bf48..14758c9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -239,10 +239,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
return new ColumnRowType(columns);
}
+ public TableRef getTableRef() {
+ return columnRowType.getColumnByIndex(0).getTableRef();
+ }
+
@SuppressWarnings("deprecation")
public TblColRef makeRewriteColumn(String name) {
- TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef();
- return tableRef.makeFakeColumn(name);
+ return getTableRef().makeFakeColumn(name);
}
public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index f1f5a48..ccb2900 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -38,7 +39,7 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPTableScan;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -73,47 +74,31 @@ public class ModelChooser {
private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) {
Map<String, String> result = Maps.newHashMap();
- // the greedy match is not perfect but works for the moment
- Map<String, List<JoinDesc>> modelJoinsMap = model.getFKSideJoinMap();
for (OLAPContext ctx : contexts) {
- for (JoinDesc queryJoin : ctx.joins) {
- String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
- List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
- if (modelJoins == null)
- return null;
-
- JoinDesc matchJoin = null;
- for (JoinDesc modelJoin : modelJoins) {
- if (modelJoin.matches(queryJoin)) {
- matchJoin = modelJoin;
- break;
- }
- }
- if (matchJoin == null)
- return null;
+ TableRef firstTable = ctx.firstTableScan.getTableRef();
- matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
- matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
- }
-
- OLAPTableScan firstTable = ctx.firstTableScan;
- String firstTableAlias = firstTable.getAlias();
- if (result.containsKey(firstTableAlias) == false) {
- TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
- if (tableRef == null)
- return null;
- matchesAdd(firstTableAlias, tableRef.getAlias(), result);
+ Map<String, String> matchUp = null;
+
+ if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) {
+ // one lookup table
+ String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias();
+ matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
+ } else {
+ // normal big joins
+ if (ctx.joinsTree == null) {
+ ctx.joinsTree = new JoinsTree(firstTable, ctx.joins);
+ }
+ matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result);
}
+
+ if (matchUp == null)
+ return null;
+
+ result.putAll(matchUp);
}
-
return result;
}
- private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) {
- String existingTarget = result.put(origAlias, targetAlias);
- Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
- }
-
private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) {
// the first context, which is the top most context, contains all columns from all contexts
OLAPContext first = contexts.get(0);