You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:23 UTC
[10/50] incubator-kylin git commit: KYLIN-624 Add dimension and
metric info into DataModel
KYLIN-624 Add dimension and metric info into DataModel
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c3c47db3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c3c47db3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c3c47db3
Branch: refs/heads/streaming
Commit: c3c47db365f8e2dfa9cb29d2537b09942c70441e
Parents: def5e7e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 16:47:50 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 16:47:50 2015 +0800
----------------------------------------------------------------------
.../test_kylin_inner_join_model_desc.json | 59 ++++++++++++++++++++
.../test_kylin_left_join_model_desc.json | 59 ++++++++++++++++++++
.../kylin/metadata/model/DataModelDesc.java | 20 +++++--
.../kylin/metadata/MetadataManagerTest.java | 19 +++++--
.../kylin/storage/hybrid/HybridInstance.java | 4 ++
5 files changed, 152 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 655045a..86f8169 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -30,6 +30,65 @@
"foreign_key" : [ "SLR_SEGMENT_CD" ]
}
} ],
+ "dimensions": [
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "lstg_format_name",
+ "LSTG_SITE_ID",
+ "SLR_SEGMENT_CD"
+ ]
+ },
+ {
+ "table": "default.test_category_groupings",
+ "columns": [
+ "leaf_categ_id",
+ "site_id",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ },
+ {
+ "table": "edw.test_sites",
+ "columns": [
+ "site_id",
+ "site_name",
+ "cre_user"
+ ]
+ },
+ {
+ "table": "edw.test_seller_type_dim",
+ "columns": [
+ "seller_type_cd",
+ "seller_type_desc"
+ ]
+ },
+ {
+ "table": "edw.test_cal_dt",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
+ },
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
+ "SELLER_ID"
+ ]
+ }
+ ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT"
+ ],
"capacity" : "MEDIUM",
"last_modified" : 1422435345352,
"fact_table" : "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index 1868c54..fa11c33 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -30,6 +30,65 @@
"foreign_key" : [ "SLR_SEGMENT_CD" ]
}
} ],
+ "dimensions": [
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "lstg_format_name",
+ "LSTG_SITE_ID",
+ "SLR_SEGMENT_CD"
+ ]
+ },
+ {
+ "table": "default.test_category_groupings",
+ "columns": [
+ "leaf_categ_id",
+ "site_id",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ },
+ {
+ "table": "edw.test_sites",
+ "columns": [
+ "site_id",
+ "site_name",
+ "cre_user"
+ ]
+ },
+ {
+ "table": "edw.test_seller_type_dim",
+ "columns": [
+ "seller_type_cd",
+ "seller_type_desc"
+ ]
+ },
+ {
+ "table": "edw.test_cal_dt",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
+ },
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
+ "SELLER_ID"
+ ]
+ }
+ ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT"
+ ],
"capacity" : "MEDIUM",
"last_modified" : 1422435345352,
"fact_table" : "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 6d1bc65..5a72386 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,10 +18,7 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
import org.apache.commons.lang.ArrayUtils;
@@ -50,6 +47,12 @@ public class DataModelDesc extends RootPersistentEntity {
@JsonProperty("lookups")
private LookupDesc[] lookups;
+ @JsonProperty("dimensions")
+ private List<DimensionDesc> dimensions;
+
+ @JsonProperty("metrics")
+ private String[] metrics;
+
@JsonProperty("filter_condition")
private String filterCondition;
@JsonProperty("partition_desc")
@@ -143,6 +146,8 @@ public class DataModelDesc extends RootPersistentEntity {
public void init(Map<String, TableDesc> tables) {
initJoinColumns(tables);
+
+ DimensionDesc.capicalizeStrings(dimensions);
}
private void initJoinColumns(Map<String, TableDesc> tables) {
@@ -218,4 +223,11 @@ public class DataModelDesc extends RootPersistentEntity {
return ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
}
+ public List<DimensionDesc> getDimensions() {
+ return dimensions;
+ }
+
+ public String[] getMetrics() {
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index e71294a..3b65d97 100644
--- a/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata;
import java.util.List;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.junit.After;
import org.junit.Assert;
@@ -28,6 +29,8 @@ import org.junit.Test;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import static org.apache.kylin.metadata.MetadataManager.*;
+
/**
* Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 2:38 PM To
* change this template use File | Settings | File Templates.
@@ -46,14 +49,14 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testListAllTables() throws Exception {
- List<TableDesc> tables = MetadataManager.getInstance(getTestConfig()).listAllTables();
+ List<TableDesc> tables = getInstance(getTestConfig()).listAllTables();
Assert.assertNotNull(tables);
Assert.assertTrue(tables.size() > 0);
}
@Test
public void testFindTableByName() throws Exception {
- TableDesc table = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT");
+ TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT");
Assert.assertNotNull(table);
Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity());
}
@@ -61,9 +64,15 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testGetInstance() throws Exception {
- Assert.assertNotNull(MetadataManager.getInstance(getTestConfig()));
- Assert.assertNotNull(MetadataManager.getInstance(getTestConfig()).listAllTables());
- Assert.assertTrue(MetadataManager.getInstance(getTestConfig()).listAllTables().size() > 0);
+ Assert.assertNotNull(getInstance(getTestConfig()));
+ Assert.assertNotNull(getInstance(getTestConfig()).listAllTables());
+ Assert.assertTrue(getInstance(getTestConfig()).listAllTables().size() > 0);
}
+ @Test
+ public void testDataModel() throws Exception {
+ DataModelDesc modelDesc = getInstance(getTestConfig()).getDataModelDesc("test_kylin_left_join_model_desc");
+
+ Assert.assertTrue(modelDesc.getDimensions().size()>0);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index ddccf90..fcdee34 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -100,6 +100,10 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
this.name = name;
}
+ @Override
+ public String toString() {
+ return getCanonicalName();
+ }
@Override
public String getCanonicalName() {