You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/24 07:35:04 UTC
[6/7] kylin git commit: KYLIN-2314 Use col identity instead of col
name in dictionary refs
KYLIN-2314 Use col identity instead of col name in dictionary refs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6c67197
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6c67197
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6c67197
Branch: refs/heads/KYLIN-2314
Commit: c6c6719730e05c66c14407cfd7926a0a54a6009a
Parents: 30d3763
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 23 18:06:58 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 24 14:46:39 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 26 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 30 +-
.../org/apache/kylin/cube/CubeDescTest.java | 40 +-
.../kylin/metadata/MetadataManagerTest.java | 67 ++-
.../kylin/metadata/model/JoinsTreeTest.java | 11 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 -
.../engine/mr/steps/CreateDictionaryJob.java | 5 +-
.../engine/mr/steps/CubingExecutableUtil.java | 8 -
.../engine/mr/steps/FactDistinctColumnsJob.java | 3 +
.../mr/steps/FactDistinctColumnsReducer.java | 39 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 81 ++--
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 -
.../localmeta/cube_desc/ci_inner_join_cube.json | 248 ++++++++---
.../localmeta/cube_desc/ci_left_join_cube.json | 248 ++++++++---
.../test_kylin_snowflake_sales_cube.json | 268 ------------
.../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++++++++++
.../model_desc/ci_inner_join_model.json | 100 ++++-
.../model_desc/ci_left_join_model.json | 100 ++++-
.../test_kylin_snowflake_sales_model.json | 87 ----
.../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 34 ++
.../localmeta/table/DEFAULT.TEST_COUNTRY.json | 24 ++
.../table/DEFAULT.TEST_KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 --
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 -------------------
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 -------
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 ---
.../java/org/apache/kylin/query/H2Database.java | 11 +-
30 files changed, 1080 insertions(+), 1295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 564a4e2..089c3ed 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -125,11 +125,15 @@ public class DeployUtil {
static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings";
static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
static final String TABLE_ORDER = "default.test_order";
+ static final String TABLE_ACCOUNT = "default.test_account";
+ static final String TABLE_COUNTRY = "default.test_country";
static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table";
static final String TABLE_SITES = "edw.test_sites";
- static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+ static final String[] TABLE_NAMES = new String[] { //
+ TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, //
+ TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT, TABLE_COUNTRY };
public static void prepareTestDataForNormalCubes(String modelName) throws Exception {
@@ -228,21 +232,15 @@ public class DeployUtil {
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase())));
+ for (String tablename : TABLE_NAMES) {
+ hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(tablename.toUpperCase())));
+ }
// load data to hive tables
// LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_ORDER, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir));
+ for (String tablename : TABLE_NAMES) {
+ hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), tableFileDir));
+ }
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_SELLER_TYPE_DIM, TABLE_SELLER_TYPE_DIM_TABLE));
@@ -251,7 +249,7 @@ public class DeployUtil {
}
private static String generateLoadDataHql(String tableName, String tableFileDir) {
- return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase();
+ return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName;
}
private static String[] generateCreateTableHql(TableDesc tableDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index fe2030a..9670b89 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -240,7 +240,7 @@ public class CubeManager implements IRealizationProvider {
if (dictInfo != null) {
Dictionary<?> dict = dictInfo.getDictionaryObject();
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId() });
+ cubeSeg.getRowkeyStats().add(new Object[] { col.getIdentity(), dict.getSize(), dict.getSizeOfId() });
CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
update.setToUpdateSegs(cubeSeg);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index e155f86..36a6044 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -98,9 +98,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
@JsonProperty("snapshots")
private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
- @JsonProperty("index_path")
- private String indexPath;
-
@JsonProperty("rowkey_stats")
private List<Object[]> rowkeyStats = Lists.newArrayList();
@@ -296,15 +293,22 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
}
public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
+ String r;
+ String dictKey = col.getIdentity();
+ r = getDictionaries().get(dictKey);
+
+ // try Kylin v1.x dict key as well
+ if (r == null) {
+ String v1DictKey = col.getTable() + "/" + col.getName();
+ r = getDictionaries().get(v1DictKey);
+ }
+
+ return r;
}
public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
+ String dictKey = col.getIdentity();
+ getDictionaries().put(dictKey, dictResPath);
}
public void setStorageLocationIdentifier(String storageLocationIdentifier) {
@@ -523,14 +527,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
return cubeInstance;
}
- public String getIndexPath() {
- return indexPath;
- }
-
- public void setIndexPath(String indexPath) {
- this.indexPath = indexPath;
- }
-
public Map<String, String> getAdditionalInfo() {
return additionalInfo;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index fb003bc..39b6433 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -30,7 +30,9 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -40,8 +42,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
/**
* @author yangli9
@@ -94,8 +95,39 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
@Test
public void testCiCube() {
CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());
- assertNotNull(mgr.getCubeDesc("ci_left_join_cube"));
- assertNotNull(mgr.getCubeDesc("ci_inner_join_cube"));
+ CubeDesc lc = mgr.getCubeDesc("ci_left_join_cube");
+ CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube");
+ assertNotNull(lc);
+ assertNotNull(ic);
+
+ // assert the two CI cubes are identical apart from the left/inner difference
+ assertEquals(lc.getDimensions().size(), ic.getDimensions().size());
+ for (int i = 0, n = lc.getDimensions().size(); i < n; i++) {
+ DimensionDesc ld = lc.getDimensions().get(i);
+ DimensionDesc id = ic.getDimensions().get(i);
+ assertEquals(ld.getTable(), id.getTable());
+ assertEquals(ld.getColumn(), id.getColumn());
+ assertArrayEquals(ld.getDerived(), id.getDerived());
+ }
+
+ assertEquals(lc.getMeasures().size(), ic.getMeasures().size());
+ for (int i = 0, n = lc.getMeasures().size(); i < n; i++) {
+ MeasureDesc lm = lc.getMeasures().get(i);
+ MeasureDesc im = ic.getMeasures().get(i);
+ assertEquals(lm.getName(), im.getName());
+ assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression());
+ assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType());
+ }
+
+ assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size());
+ for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) {
+ AggregationGroup lag = lc.getAggregationGroups().get(i);
+ AggregationGroup iag = ic.getAggregationGroups().get(i);
+ assertArrayEquals(lag.getIncludes(), iag.getIncludes());
+ assertArrayEquals(lag.getSelectRule().mandatory_dims, iag.getSelectRule().mandatory_dims);
+ assertArrayEquals(lag.getSelectRule().hierarchy_dims, iag.getSelectRule().hierarchy_dims);
+ assertArrayEquals(lag.getSelectRule().joint_dims, iag.getSelectRule().joint_dims);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index a27f4cf..b372be2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata;
import static org.apache.kylin.metadata.MetadataManager.getInstance;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,6 +27,8 @@ import java.util.List;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.junit.After;
@@ -50,8 +53,50 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testCiModel() {
MetadataManager mgr = getInstance(getTestConfig());
- Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model"));
- Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model"));
+ DataModelDesc lm = mgr.getDataModelDesc("ci_left_join_model");
+ DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
+ assertSnowflakeQuality(lm);
+ assertSnowflakeQuality(im);
+
+ // check inner/left models are identical apart from the left/inner difference
+ assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
+ for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
+ JoinTableDesc lt = lm.getJoinTables()[i];
+ JoinTableDesc it = im.getJoinTables()[i];
+ assertEquals(lt.getAlias(), it.getAlias());
+ assertEquals(lt.getKind(), it.getKind());
+ assertEquals(lt.getTable(), it.getTable());
+ assertArrayEquals(lt.getJoin().getForeignKey(), it.getJoin().getForeignKey());
+ assertArrayEquals(lt.getJoin().getPrimaryKey(), it.getJoin().getPrimaryKey());
+ assertTrue(lt.getJoin().isLeftJoin());
+ assertTrue(it.getJoin().isInnerJoin());
+ }
+
+ assertEquals(lm.getDimensions().size(), im.getDimensions().size());
+ for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
+ ModelDimensionDesc ld = lm.getDimensions().get(i);
+ ModelDimensionDesc id = im.getDimensions().get(i);
+ assertEquals(ld.getTable(), id.getTable());
+ assertArrayEquals(ld.getColumns(), id.getColumns());
+ }
+
+ assertArrayEquals(lm.getMetrics(), im.getMetrics());
+ }
+
+ private void assertSnowflakeQuality(DataModelDesc model) {
+ Assert.assertNotNull(model);
+ try {
+ model.findTable("TEST_COUNTRY");
+ Assert.fail();
+ } catch (IllegalArgumentException ex) {
+ // excepted
+ }
+
+ Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+ Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+ Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+ Assert.assertNotNull(model.findColumn("BUYER_ID"));
+
}
@Test
@@ -82,24 +127,6 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
}
@Test
- public void testSnowflakeDataModel() throws Exception {
- DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model");
- Assert.assertTrue(model.getDimensions().size() > 0);
-
- try {
- model.findTable("KYLIN_COUNTRY");
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- // excepted
- }
-
- Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
- Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
- Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
- Assert.assertNotNull(model.findColumn("BUYER_ID"));
- }
-
- @Test
public void testTableSample() throws IOException {
TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("TEST.TEST_TABLE");
Assert.assertNotNull(tableExtDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
index f52ef45..d5e9de7 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -48,21 +48,22 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() {
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ DataModelDesc model = mgr.getDataModelDesc("ci_left_join_model");
JoinsTree joinsTree = model.getJoinsTree();
Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY");
assertTrue(chain.table == model.findTable("BUYER_COUNTRY"));
assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT"));
- assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES"));
- assertTrue(chain.fkSide.fkSide.join == null);
- assertTrue(chain.fkSide.fkSide.fkSide == null);
+ assertTrue(chain.fkSide.fkSide.table == model.findTable("TEST_ORDER"));
+ assertTrue(chain.fkSide.fkSide.fkSide.table == model.findTable("TEST_KYLIN_FACT"));
+ assertTrue(chain.fkSide.fkSide.fkSide.join == null);
+ assertTrue(chain.fkSide.fkSide.fkSide.fkSide == null);
}
@Test
public void testMatch() {
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ DataModelDesc model = mgr.getDataModelDesc("ci_inner_join_model");
JoinsTree joinsTree = model.getJoinsTree();
Map<String, String> matchUp = joinsTree.matches(joinsTree);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 47eb9c3..5f5814b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -100,7 +100,6 @@ public class JobBuilderSupport {
CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
- CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams());
return result;
@@ -125,7 +124,6 @@ public class JobBuilderSupport {
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams());
- CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams());
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 4985503..8b9697e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -57,14 +57,13 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, new DistinctColumnValuesProvider() {
@Override
public ReadableTable getDistinctValuesFor(TblColRef col) {
- return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getName(), col.getType());
+ return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getIdentity(), col.getType());
}
}, new DictionaryProvider() {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path colDir = new Path(factColumnsInputPath, col.getName());
- Path dictFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString());
if (fs.exists(dictFile) == false)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
index b0d5a89..65c5869 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
@@ -135,12 +135,4 @@ public class CubingExecutableUtil {
return params.get(MERGED_STATISTICS_PATH);
}
- public static void setIndexPath(String indexPath, Map<String, String> params) {
- params.put(INDEX_PATH, indexPath);
- }
-
- public static String getIndexPath(Map<String, String> params) {
- return params.get(INDEX_PATH);
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 2eb694e..9fc8922 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -145,6 +145,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.setPartitionerClass(FactDistinctColumnPartitioner.class);
job.setNumReduceTasks(numberOfReducers);
+ // important, reducer writes HDFS directly at the moment
+ job.setReduceSpeculativeExecution(false);
+
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 0223914..5d42797 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -59,7 +59,7 @@ import com.google.common.collect.Maps;
*/
public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableKey, Text, NullWritable, Text> {
- protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
+ private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
private List<TblColRef> columnList;
private String statisticsOutput = null;
@@ -76,6 +76,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private int uhcReducerCount;
private Map<Integer, Integer> reducerIdToColumnIndex = new HashMap<>();
private int taskId;
+ private boolean isPartitionCol = false;
+ private int rowCount = 0;
//local build dict
private boolean isReducerLocalBuildDict;
@@ -84,7 +86,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private long timeMinValue = Long.MAX_VALUE;
public static final String DICT_FILE_POSTFIX = ".rldict";
public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
- private boolean isPartitionCol = false;
@Override
protected void setup(Context context) throws IOException {
@@ -111,6 +112,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
+ logger.info("Reducer " + taskId + " handling stats");
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
isStatistics = false;
@@ -120,6 +122,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
logger.info("Do not have partition col. This reducer will keep empty");
}
colValues = Lists.newLinkedList();
+ logger.info("Reducer " + taskId + " handling partition column " + col);
} else {
// normal col
isStatistics = false;
@@ -135,6 +138,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
}
+ logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict);
}
}
@@ -178,6 +182,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
} else if (isPartitionCol) {
// partition col
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ logAFewRows(value);
long time = DateFormat.stringToMillis(value);
timeMinValue = Math.min(timeMinValue, time);
timeMaxValue = Math.max(timeMaxValue, time);
@@ -185,6 +190,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// normal col
if (isReducerLocalBuildDict) {
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ logAFewRows(value);
builder.addValue(value);
} else {
colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
@@ -195,14 +201,22 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
}
+
+ rowCount++;
+ }
+
+ private void logAFewRows(String value) {
+ if (rowCount < 10) {
+ logger.info("Received value: " + value);
+ }
}
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getName());
- final String fileName = col.getName() + "-" + taskId % uhcReducerCount;
+ final Path colDir = new Path(outputPath, col.getIdentity());
+ final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
final Path outputFile = new Path(colDir, fileName);
FSDataOutputStream out = null;
@@ -229,7 +243,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
- final String fileName = col.getName() + DICT_FILE_POSTFIX;
+ final String fileName = col.getIdentity() + DICT_FILE_POSTFIX;
FSDataOutputStream out = getOutputStream(context, fileName);
try {
String dictClassName = dict.getClass().getName();
@@ -242,7 +256,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
private void outputPartitionInfo(Context context) throws IOException {
- final String fileName = col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
+ final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX;
FSDataOutputStream out = getOutputStream(context, fileName);
try {
out.writeLong(timeMinValue);
@@ -256,15 +270,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
- final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getName());
- final Path outputFile = new Path(colDir, outputFileName);
- FSDataOutputStream out = null;
- if (!fs.exists(colDir)) {
- fs.mkdirs(colDir);
+ final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
+ final Path outputFile = new Path(outputPath, outputFileName);
+ if (!fs.exists(outputPath)) {
+ fs.mkdirs(outputPath);
}
- fs.deleteOnExit(outputFile);
- out = fs.create(outputFile);
+ FSDataOutputStream out = fs.create(outputFile);
return out;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index 5692c76..ed65343 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -31,6 +31,8 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.hash.HashFunction;
@@ -41,6 +43,8 @@ import com.google.common.hash.Hashing;
*/
public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> {
+ private static final Logger logger = LoggerFactory.getLogger(FactDistinctHiveColumnsMapper.class);
+
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
@@ -51,7 +55,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private int rowCount = 0;
private int samplingPercentage;
private ByteArray[] row_hashcodes = null;
- private ByteBuffer keyBuffer;
+ private ByteBuffer tmpbuf;
private static final Text EMPTY_TEXT = new Text();
public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE;
public static final byte MARK_FOR_HLL = (byte) 0xFF;
@@ -62,7 +66,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
- keyBuffer = ByteBuffer.allocate(4096);
+ tmpbuf = ByteBuffer.allocate(4096);
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
@@ -127,55 +131,54 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException {
String[] row = flatTableInputFormat.parseMapperInput(record);
- keyBuffer.clear();
- try {
- for (int i = 0; i < factDictCols.size(); i++) {
- String fieldValue = row[dictionaryColumnIndex[i]];
- if (fieldValue == null)
- continue;
- int offset = keyBuffer.position();
-
- int reducerIndex;
- if (uhcIndex[i] == 0) {
- //for the normal dictionary column
- reducerIndex = columnIndexToReducerBeginId.get(i);
- } else {
- //for the uhc
- reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount;
- }
- keyBuffer.put(Bytes.toBytes(reducerIndex)[3]);
- keyBuffer.put(Bytes.toBytes(fieldValue));
- outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- sortableKey.setText(outputKey);
- //judge type
- sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
- context.write(sortableKey, EMPTY_TEXT);
+ for (int i = 0; i < factDictCols.size(); i++) {
+ String fieldValue = row[dictionaryColumnIndex[i]];
+ if (fieldValue == null)
+ continue;
+
+ int reducerIndex;
+ if (uhcIndex[i] == 0) {
+ //for the normal dictionary column
+ reducerIndex = columnIndexToReducerBeginId.get(i);
+ } else {
+ //for the uhc
+ reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount;
+ }
+
+ tmpbuf.clear();
+ tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
+ tmpbuf.put(Bytes.toBytes(fieldValue));
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
+ sortableKey.setText(outputKey);
+ //judge type
+ sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
+ context.write(sortableKey, EMPTY_TEXT);
+
+ // log a few rows for troubleshooting
+ if (rowCount < 10) {
+ logger.info("Sample output: " + factDictCols.get(i) + " '" + fieldValue + "' => reducer " + reducerIndex);
}
- } catch (Exception ex) {
- handleErrorRecord(row, ex);
}
if (collectStatistics) {
- if (rowCount < samplingPercentage) {
+ if (rowCount % 100 < samplingPercentage) {
putRowKeyToHLL(row);
}
if (needFetchPartitionCol == true) {
String fieldValue = row[partitionColumnIndex];
if (fieldValue != null) {
- int offset = keyBuffer.position();
- keyBuffer.put(MARK_FOR_PARTITION_COL);
- keyBuffer.put(Bytes.toBytes(fieldValue));
- outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
+ tmpbuf.clear();
+ tmpbuf.put(MARK_FOR_PARTITION_COL);
+ tmpbuf.put(Bytes.toBytes(fieldValue));
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
sortableKey.setTypeId((byte) 0);
context.write(sortableKey, EMPTY_TEXT);
}
}
}
-
- if (rowCount++ == 100)
- rowCount = 0;
+ rowCount++;
}
private void putRowKeyToHLL(String[] row) {
@@ -211,10 +214,10 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
for (int i = 0; i < cuboidIds.length; i++) {
hll = allCuboidsHLL[i];
- keyBuffer.clear();
- keyBuffer.put(MARK_FOR_HLL); // one byte
- keyBuffer.putLong(cuboidIds[i]);
- outputKey.set(keyBuffer.array(), 0, keyBuffer.position());
+ tmpbuf.clear();
+ tmpbuf.put(MARK_FOR_HLL); // one byte
+ tmpbuf.putLong(cuboidIds[i]);
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
hllBuf.clear();
hll.writeRegisters(hllBuf);
outputValue.set(hllBuf.array(), 0, hllBuf.position());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index d3becfe..dcc9190 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -59,7 +59,6 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
- segment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
@@ -81,8 +80,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
- Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index d2fa73e..add5c42 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -75,7 +75,6 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
mergedSegment.setInputRecords(sourceCount);
mergedSegment.setInputRecordsSize(sourceSize);
mergedSegment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
- mergedSegment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams()));
mergedSegment.setLastBuildTime(System.currentTimeMillis());
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 287ead6..1ebd7f3 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -5,61 +5,96 @@
"description" : null,
"dimensions" : [ {
"name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
+ "table" : "TEST_CAL_DT",
"column" : "{FK}",
"derived" : [ "WEEK_BEG_DT" ]
}, {
"name" : "ORDER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "ORDER_ID"
}, {
"name" : "TEST_DATE_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_DATE_ENC"
}, {
"name" : "TEST_TIME_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_TIME_ENC"
}, {
"name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "{FK}",
"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "META_CATEG_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL2_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL3_NAME",
"derived" : null
}, {
"name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "LSTG_FORMAT_NAME",
"derived" : null
}, {
"name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
+ "table" : "TEST_SITES",
"column" : "{FK}",
"derived" : [ "SITE_NAME", "CRE_USER" ]
}, {
"name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "table" : "TEST_SELLER_TYPE_DIM",
"column" : "{FK}",
"derived" : [ "SELLER_TYPE_DESC" ]
}, {
"name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
+ "table" : "TEST_KYLIN_FACT",
+ "column" : "SELLER_ID"
+ }, {
+ "name" : "SELLER_BUYER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "SELLER_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME"
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "TEST_ORDER",
+ "column" : "BUYER_ID"
+ }, {
+ "name" : "BUYER_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "BUYER_SELLER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME"
} ],
"measures" : [ {
"name" : "TRANS_CNT",
@@ -77,7 +112,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "ITEM_COUNT"
+ "value" : "TEST_KYLIN_FACT.ITEM_COUNT"
},
"returntype" : "bigint"
}
@@ -87,7 +122,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -97,7 +132,7 @@
"expression" : "MIN",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -107,7 +142,7 @@
"expression" : "MAX",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -117,7 +152,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
},
"returntype" : "hllc(10)"
}
@@ -127,10 +162,10 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "LSTG_FORMAT_NAME",
+ "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "hllc(10)"
@@ -141,10 +176,10 @@
"expression" : "TOP_N",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "TEST_KYLIN_FACT.PRICE",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "topn(100)",
@@ -156,7 +191,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
},
"returntype" : "bitmap"
}
@@ -166,10 +201,38 @@
"expression" : "EXTENDED_COLUMN",
"parameter": {
"type": "column",
- "value": "ORDER_ID",
+ "value": "TEST_KYLIN_FACT.ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "BUYER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.BUYER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "SELLER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_KYLIN_FACT.SELLER_ID",
"next_parameter": {
"type": "column",
- "value": "TEST_EXTENDED_COLUMN"
+ "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
}
},
"returntype": "extendedcolumn(100)"
@@ -180,7 +243,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "TRANS_ID"
+ "value" : "TEST_KYLIN_FACT.TRANS_ID"
},
"returntype" : "raw"
}
@@ -190,7 +253,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "raw"
}
@@ -200,52 +263,79 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "CAL_DT"
+ "value" : "TEST_KYLIN_FACT.CAL_DT"
},
"returntype" : "raw"
}
} ],
"dictionaries": [ {
- "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
} ],
"rowkey" : {
"rowkey_columns" : [ {
- "column" : "SELLER_ID",
+ "column" : "TEST_KYLIN_FACT.SELLER_ID",
"encoding" : "int:4"
}, {
- "column" : "ORDER_ID",
+ "column" : "TEST_KYLIN_FACT.ORDER_ID",
"encoding" : "int:4"
}, {
- "column" : "CAL_DT",
+ "column" : "TEST_KYLIN_FACT.CAL_DT",
"encoding" : "dict"
}, {
- "column" : "LEAF_CATEG_ID",
+ "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID",
"encoding" : "dict"
}, {
- "column" : "META_CATEG_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL2_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL3_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
"encoding" : "dict"
}, {
- "column" : "LSTG_FORMAT_NAME",
+ "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"encoding" : "fixed_length:12"
}, {
- "column" : "LSTG_SITE_ID",
+ "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID",
"encoding" : "dict"
}, {
- "column" : "SLR_SEGMENT_CD",
+ "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"encoding" : "dict"
}, {
- "column" : "TEST_TIME_ENC",
+ "column" : "TEST_ORDER.TEST_TIME_ENC",
"encoding" : "time"
}, {
- "column" : "TEST_DATE_ENC",
+ "column" : "TEST_ORDER.TEST_DATE_ENC",
"encoding" : "date"
+ }, {
+ "column" : "TEST_ORDER.BUYER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
} ]
},
"signature" : null,
@@ -268,26 +358,80 @@
"name" : "f3",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ]
} ]
} ]
},
"aggregation_groups" : [ {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID",
- "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ],
"select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ],
"mandatory_dims" : [ ],
- "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ]
}
}, {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME",
- "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID",
- "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+
+ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME",
+
+ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ],
"select_rule" : {
"hierarchy_dims" : [ ],
- "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
- "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ],
+
+ "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ],
+
+ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],
+
+ [ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME" ],
+
+ [ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ] ]
}
} ],
"notify_list" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index be1ea99..4048b6e 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -5,61 +5,96 @@
"description" : null,
"dimensions" : [ {
"name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
+ "table" : "TEST_CAL_DT",
"column" : "{FK}",
"derived" : [ "WEEK_BEG_DT" ]
}, {
"name" : "ORDER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "ORDER_ID"
}, {
"name" : "TEST_DATE_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_DATE_ENC"
}, {
"name" : "TEST_TIME_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_TIME_ENC"
}, {
"name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "{FK}",
"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "META_CATEG_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL2_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL3_NAME",
"derived" : null
}, {
"name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "LSTG_FORMAT_NAME",
"derived" : null
}, {
"name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
+ "table" : "TEST_SITES",
"column" : "{FK}",
"derived" : [ "SITE_NAME", "CRE_USER" ]
}, {
"name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "table" : "TEST_SELLER_TYPE_DIM",
"column" : "{FK}",
"derived" : [ "SELLER_TYPE_DESC" ]
}, {
"name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
+ "table" : "TEST_KYLIN_FACT",
+ "column" : "SELLER_ID"
+ }, {
+ "name" : "SELLER_BUYER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "SELLER_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME"
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "TEST_ORDER",
+ "column" : "BUYER_ID"
+ }, {
+ "name" : "BUYER_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "BUYER_SELLER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME"
} ],
"measures" : [ {
"name" : "TRANS_CNT",
@@ -77,7 +112,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "ITEM_COUNT"
+ "value" : "TEST_KYLIN_FACT.ITEM_COUNT"
},
"returntype" : "bigint"
}
@@ -87,7 +122,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -97,7 +132,7 @@
"expression" : "MIN",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -107,7 +142,7 @@
"expression" : "MAX",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -117,7 +152,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
},
"returntype" : "hllc(10)"
}
@@ -127,10 +162,10 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "LSTG_FORMAT_NAME",
+ "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "hllc(10)"
@@ -141,10 +176,10 @@
"expression" : "TOP_N",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "TEST_KYLIN_FACT.PRICE",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "topn(100)",
@@ -156,7 +191,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
},
"returntype" : "bitmap"
}
@@ -166,10 +201,38 @@
"expression" : "EXTENDED_COLUMN",
"parameter": {
"type": "column",
- "value": "ORDER_ID",
+ "value": "TEST_KYLIN_FACT.ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "BUYER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.BUYER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "SELLER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_KYLIN_FACT.SELLER_ID",
"next_parameter": {
"type": "column",
- "value": "TEST_EXTENDED_COLUMN"
+ "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
}
},
"returntype": "extendedcolumn(100)"
@@ -180,7 +243,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "TRANS_ID"
+ "value" : "TEST_KYLIN_FACT.TRANS_ID"
},
"returntype" : "raw"
}
@@ -190,7 +253,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "raw"
}
@@ -200,52 +263,79 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "CAL_DT"
+ "value" : "TEST_KYLIN_FACT.CAL_DT"
},
"returntype" : "raw"
}
} ],
"dictionaries": [ {
- "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
} ],
"rowkey" : {
"rowkey_columns" : [ {
- "column" : "SELLER_ID",
+ "column" : "TEST_KYLIN_FACT.SELLER_ID",
"encoding" : "int:4"
}, {
- "column" : "ORDER_ID",
+ "column" : "TEST_KYLIN_FACT.ORDER_ID",
"encoding" : "int:4"
}, {
- "column" : "CAL_DT",
+ "column" : "TEST_KYLIN_FACT.CAL_DT",
"encoding" : "dict"
}, {
- "column" : "LEAF_CATEG_ID",
+ "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID",
"encoding" : "dict"
}, {
- "column" : "META_CATEG_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL2_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL3_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
"encoding" : "dict"
}, {
- "column" : "LSTG_FORMAT_NAME",
+ "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"encoding" : "fixed_length:12"
}, {
- "column" : "LSTG_SITE_ID",
+ "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID",
"encoding" : "dict"
}, {
- "column" : "SLR_SEGMENT_CD",
+ "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"encoding" : "dict"
}, {
- "column" : "TEST_TIME_ENC",
+ "column" : "TEST_ORDER.TEST_TIME_ENC",
"encoding" : "time"
}, {
- "column" : "TEST_DATE_ENC",
+ "column" : "TEST_ORDER.TEST_DATE_ENC",
"encoding" : "date"
+ }, {
+ "column" : "TEST_ORDER.BUYER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
} ]
},
"signature" : null,
@@ -268,26 +358,80 @@
"name" : "f3",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ]
} ]
} ]
},
"aggregation_groups" : [ {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID",
- "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ],
"select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ],
"mandatory_dims" : [ ],
- "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ]
}
}, {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME",
- "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID",
- "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+
+ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME",
+
+ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ],
"select_rule" : {
"hierarchy_dims" : [ ],
- "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
- "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ],
+
+ "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ],
+
+ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],
+
+ [ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME" ],
+
+ [ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ] ]
}
} ],
"notify_list" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
deleted file mode 100644
index ee0d68c..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
+++ /dev/null
@@ -1,268 +0,0 @@
-{
- "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "last_modified" : 1451468470824,
- "name" : "test_kylin_snowflake_sales_cube",
- "model_name" : "test_kylin_snowflake_sales_model",
- "description" : null,
- "null_string" : null,
- "dimensions" : [ {
- "name" : "TRANS_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "TRANS_ID",
- "derived" : null
- }, {
- "name" : "CAL_DT",
- "table" : "SNOWTEST.KYLIN_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
- }, {
- "name" : "META_CATEG_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEG_LVL2_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEG_LVL3_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SELLER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "SELLER_ID",
- "derived" : null
- }, {
- "name" : "BUYER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "BUYER_ID",
- "derived" : null
- }, {
- "name" : "ACCOUNT_BUYER_LEVEL",
- "table" : "BUYER_ACCOUNT",
- "column" : "ACCOUNT_BUYER_LEVEL",
- "derived" : null
- }, {
- "name" : "ACCOUNT_SELLER_LEVEL",
- "table" : "SELLER_ACCOUNT",
- "column" : "ACCOUNT_SELLER_LEVEL",
- "derived" : null
- }, {
- "name" : "BUYER_COUNTRY",
- "table" : "BUYER_ACCOUNT",
- "column" : "ACCOUNT_COUNTRY",
- "derived" : null
- }, {
- "name" : "SELLER_COUNTRY",
- "table" : "SELLER_ACCOUNT",
- "column" : "ACCOUNT_COUNTRY",
- "derived" : null
- }, {
- "name" : "BUYER_COUNTRY_NAME",
- "table" : "BUYER_COUNTRY",
- "column" : "NAME",
- "derived" : null
- }, {
- "name" : "SELLER_COUNTRY_NAME",
- "table" : "SELLER_COUNTRY",
- "column" : "NAME",
- "derived" : null
- }, {
- "name" : "OPS_USER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "OPS_USER_ID",
- "derived" : null
- }, {
- "name" : "OPS_REGION",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "OPS_REGION",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- }
- }, {
- "name" : "SELLER_CNT_HLL",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- },
- "returntype" : "hllc(10)"
- }
- }, {
- "name" : "TOP_SELLER",
- "function" : {
- "expression" : "TOP_N",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
- },
- "returntype" : "topn(100)"
- }
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "BUYER_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "SELLER_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "TRANS_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "PART_DT",
- "encoding" : "date"
- }, {
- "column" : "LEAF_CATEG_ID",
- "encoding" : "dict"
- }, {
- "column" : "META_CATEG_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL2_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL3_NAME",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_COUNTRY.NAME",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_COUNTRY.NAME",
- "encoding" : "dict"
- }, {
- "column" : "LSTG_FORMAT_NAME",
- "encoding" : "dict"
- }, {
- "column" : "LSTG_SITE_ID",
- "encoding" : "dict"
- }, {
- "column" : "OPS_USER_ID",
- "encoding" : "dict"
- }, {
- "column" : "OPS_REGION",
- "encoding" : "dict"
- } ]
- },
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
- } ]
- }, {
- "name" : "F2",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
- } ]
- } ]
- },
- "aggregation_groups" : [ {
- "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
- "mandatory_dims" : [ "PART_DT" ],
- "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
- [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
- }
- }, {
- "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
- "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
- "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "joint_dims" : [ ]
- }
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "partition_date_start" : 1325376000000,
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "override_kylin_properties" : {
- "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
- }
-}