You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/22 02:39:58 UTC
[1/2] kylin git commit: KYLIN-2294 Refactor CI,
reduce cube number to 3
Repository: kylin
Updated Branches:
refs/heads/master 9dbc24bf3 -> f0fbffd58
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled b/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
deleted file mode 100644
index cab92d0..0000000
--- a/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,test_kylin_fact.CAL_DT,test_category_groupings.LEAF_CATEG_ID,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query00.sql b/kylin-it/src/test/resources/query/sql_tableau/query00.sql
index 1092b97..218d9a1 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query00.sql
@@ -18,6 +18,8 @@
select test_cal_dt.week_beg_dt, sum(test_kylin_fact.price)
from test_kylin_fact
- inner join edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner join edw.test_cal_dt as test_cal_dt on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
group by test_cal_dt.week_beg_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query10.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query10.sql b/kylin-it/src/test/resources/query/sql_tableau/query10.sql
index cf43fa7..cc5ec14 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query10.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query10.sql
@@ -20,4 +20,6 @@ SELECT "TEST_CAL_DT"."WEEK_BEG_DT" AS "none_WEEK_BEG_DT_nk",
SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
FROM "TEST_KYLIN_FACT"
inner JOIN "EDW"."TEST_CAL_DT" AS "TEST_CAL_DT" ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT")
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY "TEST_CAL_DT"."WEEK_BEG_DT"
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query11.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query11.sql b/kylin-it/src/test/resources/query/sql_tableau/query11.sql
index 7d908fd..6cfeed4 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query11.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query11.sql
@@ -20,5 +20,7 @@ SELECT COUNT(1) AS cnt_ITEM_COUNT_ok,
TEST_CAL_DT.WEEK_BEG_DT AS none_WEEK_BEG_DT_nk
FROM "TEST_KYLIN_FACT"
inner JOIN "EDW"."TEST_CAL_DT" AS "TEST_CAL_DT" ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT")
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY "TEST_CAL_DT"."WEEK_BEG_DT"
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query12.sql b/kylin-it/src/test/resources/query/sql_tableau/query12.sql
index cf43fa7..cc5ec14 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query12.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query12.sql
@@ -20,4 +20,6 @@ SELECT "TEST_CAL_DT"."WEEK_BEG_DT" AS "none_WEEK_BEG_DT_nk",
SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
FROM "TEST_KYLIN_FACT"
inner JOIN "EDW"."TEST_CAL_DT" AS "TEST_CAL_DT" ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT")
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY "TEST_CAL_DT"."WEEK_BEG_DT"
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query13.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query13.sql b/kylin-it/src/test/resources/query/sql_tableau/query13.sql
index e1b00d9..4e1faaf 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query13.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query13.sql
@@ -19,4 +19,6 @@
SELECT EXTRACT(YEAR FROM TEST_CAL_DT.WEEK_BEG_DT) AS yr_WEEK_BEG_DT_ok
FROM TEST_KYLIN_FACT
inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT ON (TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT)
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY EXTRACT(YEAR FROM TEST_CAL_DT.WEEK_BEG_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query14.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query14.sql b/kylin-it/src/test/resources/query/sql_tableau/query14.sql
index 50d8952..23c790c 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query14.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query14.sql
@@ -16,4 +16,9 @@
-- limitations under the License.
--
-SELECT QUARTER("TEST_CAL_DT"."WEEK_BEG_DT") AS "qr_WEEK_BEG_DT_ok", EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "yr_WEEK_BEG_DT_ok" FROM "TEST_KYLIN_FACT" inner JOIN "EDW"."TEST_CAL_DT" AS "TEST_CAL_DT" ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT") GROUP BY QUARTER("TEST_CAL_DT"."WEEK_BEG_DT"), EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT")
+SELECT QUARTER("TEST_CAL_DT"."WEEK_BEG_DT") AS "qr_WEEK_BEG_DT_ok", EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "yr_WEEK_BEG_DT_ok"
+FROM "TEST_KYLIN_FACT"
+inner JOIN "EDW"."TEST_CAL_DT" AS "TEST_CAL_DT" ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT")
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+GROUP BY QUARTER("TEST_CAL_DT"."WEEK_BEG_DT"), EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT")
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query15.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query15.sql b/kylin-it/src/test/resources/query/sql_tableau/query15.sql
index 8f30460..198413c 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query15.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query15.sql
@@ -19,4 +19,6 @@
SELECT QUARTER(TEST_CAL_DT.WEEK_BEG_DT) AS qr_WEEK_BEG_DT_ok
FROM TEST_KYLIN_FACT
inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT ON (TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT)
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY QUARTER(TEST_CAL_DT.WEEK_BEG_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query16.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query16.sql b/kylin-it/src/test/resources/query/sql_tableau/query16.sql
index ad82c09..7140d8e 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query16.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query16.sql
@@ -19,5 +19,7 @@
SELECT EXTRACT(YEAR FROM TEST_CAL_DT.WEEK_BEG_DT) AS yr_WEEK_BEG_DT_ok, QUARTER(TEST_CAL_DT.WEEK_BEG_DT) AS qr_WEEK_BEG_DT_ok
FROM TEST_KYLIN_FACT
inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT ON (TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT)
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
GROUP BY EXTRACT(YEAR FROM TEST_CAL_DT.WEEK_BEG_DT), QUARTER(TEST_CAL_DT.WEEK_BEG_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_tableau/query17.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query17.sql b/kylin-it/src/test/resources/query/sql_tableau/query17.sql
index 11f6df1..5944ff9 100644
--- a/kylin-it/src/test/resources/query/sql_tableau/query17.sql
+++ b/kylin-it/src/test/resources/query/sql_tableau/query17.sql
@@ -16,4 +16,9 @@
-- limitations under the License.
--
-SELECT EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "mn_WEEK_BEG_DT_ok", (( EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") * 100) + EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")) AS "my_WEEK_BEG_DT_ok", QUARTER("TEST_CAL_DT"."WEEK_BEG_DT") AS "qr_WEEK_BEG_DT_ok", EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "yr_WEEK_BEG_DT_ok" FROM "TEST_KYLIN_FACT" inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT") GROUP BY EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT"), QUARTER("TEST_CAL_DT"."WEEK_BEG_DT"), (( EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") * 100) + EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")), EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")
+SELECT EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "mn_WEEK_BEG_DT_ok", (( EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") * 100) + EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")) AS "my_WEEK_BEG_DT_ok", QUARTER("TEST_CAL_DT"."WEEK_BEG_DT") AS "qr_WEEK_BEG_DT_ok", EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") AS "yr_WEEK_BEG_DT_ok"
+FROM "TEST_KYLIN_FACT"
+inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT ON ("TEST_KYLIN_FACT"."CAL_DT" = "TEST_CAL_DT"."CAL_DT")
+ inner join test_category_groupings AS test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+GROUP BY EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT"), QUARTER("TEST_CAL_DT"."WEEK_BEG_DT"), (( EXTRACT(YEAR FROM "TEST_CAL_DT"."WEEK_BEG_DT") * 100) + EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")), EXTRACT(MONTH FROM "TEST_CAL_DT"."WEEK_BEG_DT")
[2/2] kylin git commit: KYLIN-2294 Refactor CI,
reduce cube number to 3
Posted by li...@apache.org.
KYLIN-2294 Refactor CI, reduce cube number to 3
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0fbffd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0fbffd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0fbffd5
Branch: refs/heads/master
Commit: f0fbffd58a37aa9de22b887e385502b7baf670ea
Parents: 9dbc24b
Author: Li Yang <li...@apache.org>
Authored: Mon Dec 19 18:03:37 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 22 10:38:15 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 9 +-
.../apache/kylin/common/KylinConfigBase.java | 5 -
.../kylin/common/persistence/ResourceStore.java | 2 +-
.../org/apache/kylin/cube/CubeDescTest.java | 8 +
.../validation/rule/DictionaryRuleTest.java | 6 +-
.../kylin/cube/project/ProjectManagerTest.java | 4 +-
.../kylin/source/datagen/ColumnGenerator.java | 14 +-
.../source/datagen/ModelDataGenerator.java | 31 +-
.../kylin/measure/hllc/HLLCounterOldTest.java | 2 +-
.../kylin/measure/hllc/HLLCounterTest.java | 2 +-
.../kylin/metadata/MetadataManagerTest.java | 7 +
.../mr/steps/FactDistinctColumnsReducer.java | 3 +
.../localmeta/cube/ci_inner_join_cube.json | 10 +
.../localmeta/cube/ci_left_join_cube.json | 10 +
.../localmeta/cube_desc/ci_inner_join_cube.json | 286 +++++++++++++++++++
.../localmeta/cube_desc/ci_left_join_cube.json | 286 +++++++++++++++++++
...t_kylin_cube_without_slr_left_join_desc.json | 4 +-
.../localmeta/hybrid/ci_inner_join_hybrid.json | 14 +
.../hybrid/test_kylin_hybrid_inner_join.json | 14 -
.../model_desc/ci_inner_join_model.json | 139 +++++++++
.../model_desc/ci_left_join_model.json | 141 +++++++++
.../test_kylin_inner_join_model_desc.json | 7 +-
.../test_kylin_left_join_model_desc.json | 7 +-
.../localmeta/project/default.json | 92 +++---
.../table/DEFAULT.TEST_KYLIN_FACT.json | 39 ++-
.../localmeta/table/DEFAULT.TEST_ORDER.json | 18 ++
.../sandbox/kylin_job_conf_inmem.xml | 6 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 4 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 2 +-
.../kylin/provision/BuildCubeWithEngine.java | 99 ++-----
.../apache/kylin/query/ITCombinationTest.java | 15 +-
.../apache/kylin/query/ITKylinQueryTest.java | 29 +-
.../resources/query/sql_derived/query12.sql | 2 +-
.../query/sql_distinct_precisely/query00.sql | 2 +-
.../query/sql_distinct_precisely/query01.sql | 2 +-
.../query/sql_distinct_precisely/query02.sql | 2 +-
.../query/sql_distinct_precisely/query03.sql | 2 +-
.../query/sql_distinct_precisely/query04.sql | 2 +-
.../query/sql_intersect_count/query00.sql | 14 +-
.../test/resources/query/sql_limit/query01.sql | 10 +-
.../test/resources/query/sql_raw/query04.sql | 2 +-
.../test/resources/query/sql_raw/query06.sql | 2 +-
.../test/resources/query/sql_raw/query23.sql | 2 +-
.../test/resources/query/sql_raw/query24.sql | 24 --
.../query/sql_raw/query26.sql.disabled | 24 --
.../resources/query/sql_tableau/query00.sql | 4 +-
.../resources/query/sql_tableau/query10.sql | 2 +
.../resources/query/sql_tableau/query11.sql | 2 +
.../resources/query/sql_tableau/query12.sql | 2 +
.../resources/query/sql_tableau/query13.sql | 2 +
.../resources/query/sql_tableau/query14.sql | 7 +-
.../resources/query/sql_tableau/query15.sql | 2 +
.../resources/query/sql_tableau/query16.sql | 2 +
.../resources/query/sql_tableau/query17.sql | 7 +-
54 files changed, 1136 insertions(+), 299 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index c8b0d43..564a4e2 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -124,13 +124,14 @@ public class DeployUtil {
static final String TABLE_CAL_DT = "edw.test_cal_dt";
static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings";
static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
+ static final String TABLE_ORDER = "default.test_order";
static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table";
static final String TABLE_SITES = "edw.test_sites";
- static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+ static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
- public static void prepareTestDataForNormalCubes(String cubeName) throws Exception {
+ public static void prepareTestDataForNormalCubes(String modelName) throws Exception {
boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
if (!buildCubeUsingProvidedData) {
@@ -138,7 +139,7 @@ public class DeployUtil {
// data is generated according to cube descriptor and saved in resource store
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc");
+ DataModelDesc model = mgr.getDataModelDesc(modelName);
ModelDataGenerator gen = new ModelDataGenerator(model, 10000);
gen.generate();
} else {
@@ -230,6 +231,7 @@ public class DeployUtil {
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase())));
+ hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase())));
@@ -238,6 +240,7 @@ public class DeployUtil {
hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir));
+ hiveClient.executeHQL(generateLoadDataHql(TABLE_ORDER, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir));
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 01d1d36..11e48ac 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -496,11 +496,6 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
}
- @Deprecated
- public String getCreateFlatHiveTableMethod() {
- return getOptional("kylin.source.hive.create-flat-table-method", "1");
- }
-
// ============================================================================
// STORAGE.HBASE
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 466d9d2..0580576 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -109,7 +109,7 @@ abstract public class ResourceStore {
final protected KylinConfig kylinConfig;
- public ResourceStore(KylinConfig kylinConfig) {
+ protected ResourceStore(KylinConfig kylinConfig) {
this.kylinConfig = kylinConfig;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 1daa748..fb003bc 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -41,6 +41,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
/**
* @author yangli9
@@ -91,6 +92,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
}
@Test
+ public void testCiCube() {
+ CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());
+ assertNotNull(mgr.getCubeDesc("ci_left_join_cube"));
+ assertNotNull(mgr.getCubeDesc("ci_inner_join_cube"));
+ }
+
+ @Test
public void testGoodInit() throws Exception {
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
cubeDesc.init(getTestConfig());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index b6e0bcb..c272bb7 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -72,8 +72,8 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
@Test
public void testBadDesc() throws IOException {
- testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, "FakeBuilderClass"));
- testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, GlobalDictionaryBuilder.class.getName()));
+ testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, "FakeBuilderClass"));
+ testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, GlobalDictionaryBuilder.class.getName()));
}
@Test
@@ -89,7 +89,7 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
@Test
public void testBadDesc4() throws IOException {
testDictionaryDesc(ERROR_TRANSITIVE_REUSE,
- DictionaryDesc.create("lstg_site_id", "USER_ID", null),
+ DictionaryDesc.create("lstg_site_id", "SELLER_ID", null),
DictionaryDesc.create("price", "lstg_site_id", null));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
index 9728e52..a0fb4a3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
@@ -147,7 +147,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
+ CubeInstance cube = cubeMgr.getCube("ci_left_join_cube");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
assertTrue(prjMgr.listAllRealizations("default").contains(cube));
@@ -162,7 +162,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
+ CubeInstance cube = cubeMgr.getCube("ci_left_join_cube");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
index 775f4fc..7c54714 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -250,8 +250,8 @@ public class ColumnGenerator {
public String next() {
if (values.isEmpty())
return null;
- else
- return values.get(rand.nextInt(values.size()));
+
+ return values.get(rand.nextInt(values.size()));
}
}
@@ -308,7 +308,15 @@ public class ColumnGenerator {
@Override
public String next() {
- return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next();
+ String r = nullStr;
+ if (input.hasNext()) {
+ r = input.next();
+ }
+
+ if (rand.nextDouble() < nullPct) {
+ r = nullStr;
+ }
+ return r;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
index 60f2736..f1e6d9f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.source.datagen;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -34,8 +35,10 @@ import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -56,10 +59,14 @@ public class ModelDataGenerator {
boolean outprint = false; // for debug
public ModelDataGenerator(DataModelDesc model, int nRows) {
- this(model, nRows, ResourceStore.getStore(model.getConfig()), "/data");
+ this(model, nRows, ResourceStore.getStore(model.getConfig()));
}
- public ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) {
+ private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore) {
+ this(model, nRows, outputStore, "/data");
+ }
+
+ private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) {
this.model = model;
this.targetRows = nRows;
this.outputStore = outputStore;
@@ -268,8 +275,8 @@ public class ModelDataGenerator {
}
private void saveResource(byte[] content, String path) throws IOException {
+ System.out.println("Generated " + outputStore.getReadableResourcePath(path));
if (outprint) {
- System.out.println("Generated " + path);
System.out.println(Bytes.toString(content));
}
outputStore.putResource(path, new ByteArrayInputStream(content), System.currentTimeMillis());
@@ -287,4 +294,22 @@ public class ModelDataGenerator {
return model;
}
+ public static void main(String[] args) throws IOException {
+ String modelName = args[0];
+ int nRows = Integer.parseInt(args[1]);
+ String outputDir = args.length > 2 ? args[2] : null;
+
+ KylinConfig conf = KylinConfig.getInstanceFromEnv();
+ DataModelDesc model = MetadataManager.getInstance(conf).getDataModelDesc(modelName);
+ ResourceStore store = outputDir == null ? ResourceStore.getStore(conf) : ResourceStore.getStore(mockup(outputDir));
+
+ ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store);
+ gen.generate();
+ }
+
+ private static KylinConfig mockup(String outputDir) {
+ KylinConfig mockup = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
+ mockup.setMetadataUrl(new File(outputDir).getAbsolutePath());
+ return mockup;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
index c4a97cd..427d9a7 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
@@ -157,7 +157,7 @@ public class HLLCounterOldTest {
public void mergeTest() throws IOException {
double error = 0;
int n = 100;
- for (int i = 0; i < n; i++) {
+ for (int i = 0; i < n; i += 10) {
double e = merge(i);
error += e;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
index 26ad4a7..cd75eac 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
@@ -82,7 +82,7 @@ public class HLLCounterTest {
public void mergeTest() throws IOException {
double error = 0;
int n = 100;
- for (int i = 0; i < n; i++) {
+ for (int i = 0; i < n; i += 10) {
double e = merge(i);
error += e;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index 204b4e0..a27f4cf 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -46,6 +46,13 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
public void after() throws Exception {
this.cleanupTestMetadata();
}
+
+ @Test
+ public void testCiModel() {
+ MetadataManager mgr = getInstance(getTestConfig());
+ Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model"));
+ Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model"));
+ }
@Test
public void testListAllTables() throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 3115fe4..0223914 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -128,6 +128,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
+ if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
+ isReducerLocalBuildDict = false;
+ }
if (col != null && isReducerLocalBuildDict) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
new file mode 100644
index 0000000..f055be0
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
@@ -0,0 +1,10 @@
+{
+ "uuid" : "8372c3b7-a33e-4b69-83dd-0bb8b1f8117e",
+ "last_modified" : 0,
+ "name" : "ci_inner_join_cube",
+ "owner" : null,
+ "descriptor" : "ci_inner_join_cube",
+ "segments" : [ ],
+ "status" : "DISABLED",
+ "create_time" : null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
new file mode 100644
index 0000000..c3fddf0
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
@@ -0,0 +1,10 @@
+{
+ "uuid" : "56e1a32a-a33e-4b69-83dd-0bb8b1f80354",
+ "last_modified" : 0,
+ "name" : "ci_left_join_cube",
+ "owner" : null,
+ "descriptor" : "ci_left_join_cube",
+ "segments" : [ ],
+ "status" : "DISABLED",
+ "create_time" : null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
new file mode 100644
index 0000000..48a051d
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -0,0 +1,286 @@
+{
+ "uuid" : "3819ad72-3929-4dff-b59d-cd89a01238af",
+ "name" : "ci_inner_join_cube",
+ "model_name" : "ci_inner_join_model",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "ORDER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "ORDER_ID"
+ }, {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "SELLER_ID",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1"
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT"
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "SELLER_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "SELLER_FORMAT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ }
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ }
+ },
+ "returntype" : "topn(100)",
+ "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ }
+ }, {
+ "name" : "TEST_COUNT_DISTINCT_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ },
+ "returntype" : "bitmap"
+ }
+ }, {
+ "name" : "TEST_EXTENDED_COLUMN",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "TRANS_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "TRANS_ID"
+ },
+ "returntype" : "raw"
+ }
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "raw"
+ }
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT"
+ },
+ "returntype" : "raw"
+ }
+ } ],
+ "dictionaries": [ {
+ "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "SELLER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "ORDER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "CAL_DT",
+ "encoding" : "dict"
+ }, {
+ "column" : "LEAF_CATEG_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "META_CATEG_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL2_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL3_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "LSTG_SITE_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "SLR_SEGMENT_CD",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ]
+ } ]
+ }, {
+ "name" : "f3",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ }
+ }, {
+ "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ],
+ "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties": {
+ "kylin.cube.algorithm": "LAYER"
+ },
+ "partition_date_start": 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
new file mode 100644
index 0000000..df370f7
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -0,0 +1,286 @@
+{
+ "uuid" : "629ab7a8-3929-4dff-b59d-2100aadccd1a",
+ "name" : "ci_left_join_cube",
+ "model_name" : "ci_left_join_model",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "ORDER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "ORDER_ID"
+ }, {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "SELLER_ID",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1"
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT"
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "SELLER_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "SELLER_FORMAT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ }
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID"
+ }
+ },
+ "returntype" : "topn(100)",
+ "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ }
+ }, {
+ "name" : "TEST_COUNT_DISTINCT_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ },
+ "returntype" : "bitmap"
+ }
+ }, {
+ "name" : "TEST_EXTENDED_COLUMN",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "TRANS_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "TRANS_ID"
+ },
+ "returntype" : "raw"
+ }
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE"
+ },
+ "returntype" : "raw"
+ }
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT"
+ },
+ "returntype" : "raw"
+ }
+ } ],
+ "dictionaries": [ {
+ "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "SELLER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "ORDER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "CAL_DT",
+ "encoding" : "dict"
+ }, {
+ "column" : "LEAF_CATEG_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "META_CATEG_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL2_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL3_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "LSTG_SITE_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "SLR_SEGMENT_CD",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ]
+ } ]
+ }, {
+ "name" : "f3",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ }
+ }, {
+ "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ],
+ "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties": {
+ "kylin.cube.algorithm": "INMEM"
+ },
+ "partition_date_start": 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 9c26be9..9aeaa2e 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -121,7 +121,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "USER_ID",
+ "value" : "ORDER_ID",
"next_parameter" : null
},
"returntype" : "bitmap"
@@ -210,7 +210,7 @@
"dependent_measure_ref" : null
} ],
"dictionaries": [ {
- "column": "USER_ID",
+ "column": "ORDER_ID",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
} ],
"rowkey" : {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
new file mode 100644
index 0000000..05d30c5
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
@@ -0,0 +1,14 @@
+{
+ "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
+
+ "name": "ci_inner_join_hybrid",
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "ci_inner_join_cube"
+ }
+ ],
+ "cost": 100,
+ "last_modified": 1420016227424,
+ "create_time": null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
deleted file mode 100644
index 443c0a3..0000000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
-
- "name": "test_kylin_hybrid_inner_join",
- "realizations": [
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- }
- ],
- "cost": 100,
- "last_modified": 1420016227424,
- "create_time": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
new file mode 100644
index 0000000..cf6b56b
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -0,0 +1,139 @@
+{
+ "uuid": "72ab4ee2-2cdb-4b07-b39e-4c298563ae27",
+ "name": "ci_inner_join_model",
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "join_tables": [
+ {
+ "table": "DEFAULT.TEST_ORDER",
+ "kind": "FACT",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "TEST_ORDER.ORDER_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.ORDER_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "TEST_CAL_DT.CAL_DT"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.CAL_DT"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID",
+ "TEST_CATEGORY_GROUPINGS.SITE_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "TEST_SITES.SITE_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
+ ]
+ }
+ }
+ ],
+ "dimensions": [
+ {
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "columns": [
+ "TRANS_ID",
+ "ORDER_ID",
+ "CAL_DT",
+ "LSTG_FORMAT_NAME",
+ "LSTG_SITE_ID",
+ "LEAF_CATEG_ID",
+ "SLR_SEGMENT_CD",
+ "SELLER_ID"
+ ]
+ },
+ {
+ "table": "DEFAULT.TEST_ORDER",
+ "columns": [
+ "ORDER_ID",
+ "BUYER_ID"
+ ]
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "columns": [
+ "LEAF_CATEG_ID",
+ "SITE_ID",
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "columns": [
+ "SITE_ID",
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "columns": [
+ "SELLER_TYPE_CD",
+ "SELLER_TYPE_DESC"
+ ]
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "columns": [
+ "CAL_DT",
+ "WEEK_BEG_DT"
+ ]
+ }
+ ],
+ "metrics": [
+ "TEST_KYLIN_FACT.PRICE",
+ "TEST_KYLIN_FACT.ITEM_COUNT"
+ ],
+ "last_modified": 1422435345352,
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.CAL_DT",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
new file mode 100644
index 0000000..a3088ec
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -0,0 +1,141 @@
+{
+ "uuid": "ed524ee2-2cdb-4b07-b39e-4c298563a863",
+ "name": "ci_left_join_model",
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "join_tables": [
+ {
+ "table": "DEFAULT.TEST_ORDER",
+ "kind": "FACT",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "TEST_ORDER.ORDER_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.ORDER_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "TEST_CAL_DT.CAL_DT"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.CAL_DT"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID",
+ "TEST_CATEGORY_GROUPINGS.SITE_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "TEST_SITES.SITE_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
+ ]
+ }
+ }
+ ],
+ "dimensions": [
+ {
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "columns": [
+ "TRANS_ID",
+ "ORDER_ID",
+ "CAL_DT",
+ "LSTG_FORMAT_NAME",
+ "LSTG_SITE_ID",
+ "LEAF_CATEG_ID",
+ "SLR_SEGMENT_CD",
+ "SELLER_ID",
+ "TEST_EXTENDED_COLUMN",
+ "TEST_COUNT_DISTINCT_BITMAP"
+ ]
+ },
+ {
+ "table": "DEFAULT.TEST_ORDER",
+ "columns": [
+ "ORDER_ID",
+ "BUYER_ID"
+ ]
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "columns": [
+ "LEAF_CATEG_ID",
+ "SITE_ID",
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "columns": [
+ "SITE_ID",
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "columns": [
+ "SELLER_TYPE_CD",
+ "SELLER_TYPE_DESC"
+ ]
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "columns": [
+ "CAL_DT",
+ "WEEK_BEG_DT"
+ ]
+ }
+ ],
+ "metrics": [
+ "TEST_KYLIN_FACT.PRICE",
+ "TEST_KYLIN_FACT.ITEM_COUNT"
+ ],
+ "last_modified": 1422435345352,
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.CAL_DT",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 16ce8ae..49758dd 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -63,7 +63,8 @@
"TRANS_ID",
"CAL_DT",
"LEAF_CATEG_ID",
- "SELLER_ID"
+ "SELLER_ID",
+ "ORDER_ID"
]
},
{
@@ -105,9 +106,7 @@
],
"metrics": [
"PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "USER_ID"
+ "ITEM_COUNT"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index 92b336f..e4cab89 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -63,7 +63,8 @@
"TRANS_ID",
"CAL_DT",
"LEAF_CATEG_ID",
- "SELLER_ID"
+ "SELLER_ID",
+ "ORDER_ID"
]
},
{
@@ -105,9 +106,7 @@
],
"metrics": [
"PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "USER_ID"
+ "ITEM_COUNT"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 5eefce7..1ecc7cc 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -1,57 +1,49 @@
-{
- "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
- "name": "default",
- "realizations": [
- {
- "name": "test_kylin_cube_with_slr_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_empty"
- },
- {
- "name": "test_kylin_cube_without_slr_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- },
- {
- "name": "test_kylin_cube_with_slr_left_join_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- {
- "name": "test_kylin_cube_without_slr_left_join_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_left_join_empty"
- },
- {
- "name": "test_streaming_table_cube",
- "type": "CUBE",
- "realization": "test_streaming_table_cube"
- },
- {
- "name": "test_kylin_hybrid_inner_join",
- "type": "HYBRID",
- "realization": "test_kylin_hybrid_inner_join"
- },
+{
+ "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
+ "name": "default",
+ "realizations": [
+ {
+ "name": "ci_left_join_cube",
+ "type": "CUBE",
+ "realization": "ci_left_join_cube"
+ },
+ {
+ "name": "ci_inner_join_cube",
+ "type": "CUBE",
+ "realization": "ci_inner_join_cube"
+ },
+ {
+ "name": "test_streaming_table_cube",
+ "type": "CUBE",
+ "realization": "test_streaming_table_cube"
+ },
+ {
+ "name": "ci_inner_join_hybrid",
+ "type": "HYBRID",
+ "realization": "ci_inner_join_hybrid"
+ },
{
"name": "ssb",
"type": "CUBE",
"realization": "ssb"
- }
- ],
- "tables": [
- "DEFAULT.TEST_KYLIN_FACT",
- "EDW.TEST_CAL_DT",
- "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "EDW.TEST_SITES",
- "EDW.TEST_SELLER_TYPE_DIM",
- "EDW.TEST_SELLER_TYPE_DIM_TABLE",
- "DEFAULT.STREAMING_TABLE"
- ],
- "models": [
- "ssb",
- "test_kylin_inner_join_model_desc",
- "test_kylin_left_join_model_desc",
- "test_streaming_table_model_desc"
+ }
+ ],
+ "tables": [
+ "DEFAULT.TEST_KYLIN_FACT",
+ "DEFAULT.TEST_ORDER",
+ "EDW.TEST_CAL_DT",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "EDW.TEST_SITES",
+ "EDW.TEST_SELLER_TYPE_DIM",
+ "DEFAULT.STREAMING_TABLE"
+ ],
+ "models": [
+ "ssb",
+ "ci_inner_join_model",
+ "ci_left_join_model",
+ "test_kylin_inner_join_model_desc",
+ "test_kylin_left_join_model_desc",
+ "test_streaming_table_model_desc"
],
"override_kylin_properties" :{
"kylin.storage.hbase.owner-tag": "kylin@kylin.apache.org"
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 5fb3edf..1aafaca 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -9,58 +9,57 @@
"data_gen" : "ID"
}, {
"id" : "2",
+ "name" : "ORDER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "3",
"name" : "CAL_DT",
"datatype" : "date",
"data_gen" : "FK,order"
}, {
- "id" : "3",
+ "id" : "4",
"name" : "LSTG_FORMAT_NAME",
"datatype" : "string",
"data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others"
}, {
- "id" : "4",
+ "id" : "5",
"name" : "LEAF_CATEG_ID",
"datatype" : "bigint",
"data_gen" : "FK,null,nullstr=0"
}, {
- "id" : "5",
+ "id" : "6",
"name" : "LSTG_SITE_ID",
"datatype" : "int"
}, {
- "id" : "6",
+ "id" : "7",
"name" : "SLR_SEGMENT_CD",
"datatype" : "smallint",
"data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD"
}, {
- "id" : "7",
+ "id" : "8",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint",
+ "data_gen" : "RAND||10000000|10001000"
+ }, {
+ "id" : "9",
"name" : "PRICE",
"datatype" : "decimal",
"data_gen" : "RAND|.##|-100|1000"
}, {
- "id" : "8",
+ "id" : "10",
"name" : "ITEM_COUNT",
"datatype" : "int",
"data_gen" : "RAND"
}, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint",
- "data_gen" : "RAND||10000000|10001000"
- }, {
- "id" : "10",
- "name" : "USER_ID",
- "datatype" : "varchar(32)",
- "data_gen" : "RAND,order"
- }, {
"id" : "11",
- "name" : "BUYER_COUNTRY",
+ "name" : "TEST_EXTENDED_COLUMN",
"datatype" : "string",
- "data_gen" : "CN|DE|FR|JP|UK|US"
+ "data_gen" : "RAND"
}, {
"id" : "12",
- "name" : "SELLER_COUNTRY",
+ "name" : "TEST_COUNT_DISTINCT_BITMAP",
"datatype" : "string",
- "data_gen" : "CN|DE|FR|JP|UK|US"
+ "data_gen" : "RAND"
} ],
"database" : "DEFAULT",
"last_modified" : 0
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
new file mode 100644
index 0000000..05647e6
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
@@ -0,0 +1,18 @@
+{
+ "uuid" : "f216e39e-40d7-44c2-8fa2-41b3656398ae",
+ "name" : "TEST_ORDER",
+ "data_gen" : "0.5",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "ORDER_ID",
+ "datatype" : "bigint",
+ "data_gen" : "ID"
+ }, {
+ "id" : "2",
+ "name" : "BUYER_ID",
+ "datatype" : "bigint",
+ "data_gen" : "RAND||10000500|10001500"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
index 9e8fc84..b05495f 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
@@ -78,19 +78,19 @@
<!--Additional config for in-mem cubing, giving mapper more memory -->
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>768</value>
<description></description>
</property>
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx384m</value>
+ <value>-Xmx700m</value>
<description></description>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>100</value>
+ <value>10</value>
<description></description>
</property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 8827dff..806bfab 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -57,8 +57,8 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(ITDoggedCubeBuilderTest.class);
- private static final int INPUT_ROWS = 10000;
- private static final int SPLIT_ROWS = 5000;
+ private static final int INPUT_ROWS = 4000;
+ private static final int SPLIT_ROWS = 2000;
private static final int THREADS = 4;
private static CubeInstance cube;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index dea9f2d..f9b6ef2 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -85,7 +85,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
@Test
public void testKylinCube() throws Exception {
testBuild("test_kylin_cube_without_slr_left_join_empty", //
- LOCALMETA_TEST_DATA + "/data/flatten_data_for_without_slr_left_join.csv", 70000, 4);
+ LOCALMETA_TEST_DATA + "/data/flatten_data_for_without_slr_left_join.csv", 7000, 4);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index bfbeb70..31a5cee 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -76,6 +76,8 @@ public class BuildCubeWithEngine {
private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithEngine.class);
public static void main(String[] args) throws Exception {
+ long start = System.currentTimeMillis();
+ int exitCode = 0;
try {
beforeClass();
@@ -86,11 +88,15 @@ public class BuildCubeWithEngine {
logger.info("Build is done");
afterClass();
logger.info("Going to exit");
- System.exit(0);
} catch (Throwable e) {
logger.error("error", e);
- System.exit(1);
+ exitCode = 1;
}
+
+ long millis = System.currentTimeMillis() - start;
+ System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
+
+ System.exit(exitCode);
}
public static void beforeClass() throws Exception {
@@ -161,7 +167,7 @@ public class BuildCubeWithEngine {
}
public void build() throws Exception {
- DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty");
+ DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model");
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f);
testInner();
testLeft();
@@ -184,12 +190,12 @@ public class BuildCubeWithEngine {
}
private void testInner() throws Exception {
- String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" };
+ String[] testCase = new String[] { "testInnerJoinCube" };
runTestAndAssertSucceed(testCase);
}
private void testLeft() throws Exception {
- String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" };
+ String[] testCase = new String[] { "testLeftJoinCube" };
runTestAndAssertSucceed(testCase);
}
@@ -243,47 +249,24 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
// called by reflection
- private boolean testInnerJoinCubeWithSlr() throws Exception {
- final String cubeName = "test_kylin_cube_with_slr_empty";
- clearSegment(cubeName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
-
- if (fastBuildMode) {
- return buildSegment(cubeName, date1, date3);
- } else {
- if (buildSegment(cubeName, date1, date2) == true) {
- return buildSegment(cubeName, date2, date3);//empty segment
- }
- }
- return false;
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testLeftJoinCubeWithSlr() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ private boolean testLeftJoinCube() throws Exception {
+ String cubeName = "ci_left_join_cube";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
+ long date1 = 0;
long date2 = f.parse("2013-01-01").getTime();
long date3 = f.parse("2013-07-01").getTime();
long date4 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
if (fastBuildMode) {
return buildSegment(cubeName, date1, date4);
} else {
if (buildSegment(cubeName, date1, date2) == true) {
if (buildSegment(cubeName, date2, date3) == true) {
if (buildSegment(cubeName, date3, date4) == true) {
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ return mergeSegment(cubeName, date1, date3); // don't merge all segments
}
}
}
@@ -293,61 +276,25 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
// called by reflection
- private boolean testInnerJoinCubeWithoutSlr() throws Exception {
+ private boolean testInnerJoinCube() throws Exception {
- final String cubeName = "test_kylin_cube_without_slr_empty";
+ String cubeName = "ci_inner_join_cube";
clearSegment(cubeName);
+
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2013-07-01").getTime();
- long date4 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
+ long date2 = f.parse("2022-07-01").getTime();
+ long date3 = f.parse("2023-01-01").getTime();
if (fastBuildMode) {
- return buildSegment(cubeName, date1, date4);
- } else {
- if (buildSegment(cubeName, date1, date2) == true) {
- if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) {
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
- }
- }
- }
- }
- return false;
-
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testLeftJoinCubeWithoutSlr() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- clearSegment(cubeName);
-
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
- long date2 = f.parse("2012-06-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- long date4 = f.parse("2023-01-01").getTime();
-
- if (fastBuildMode) {
- return buildSegment(cubeName, date1, date4);
+ return buildSegment(cubeName, date1, date3);
} else {
- if (buildSegment(cubeName, date1, date2) == true) {
- if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) { //empty segment
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
- }
- }
+ if (buildSegment(cubeName, date1, date2) == true) { // all-in-one build
+ return buildSegment(cubeName, date2, date3); // empty segment
}
}
-
return false;
-
}
private void clearSegment(String cubeName) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 5f1e3a9..23eadaf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -63,17 +63,14 @@ public class ITCombinationTest extends ITKylinQueryTest {
@Parameterized.Parameters
public static Collection<Object[]> configs() {
return Arrays.asList(new Object[][] { //
- { "inner", "on", "v2", true }, //
- { "left", "on", "v2", true }, //
- //{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection
- //{ "left", "on", "v1", false }, // exclude view to simply model/cube selection
- //{ "left", "on", "v2", false }, // exclude view to simply model/cube selection
+ { "inner", "on", "v2" }, //
+ { "left", "on", "v2" }, //
});
}
- public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception {
+ public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine) throws Exception {
- printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes);
+ printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine);
ITKylinQueryTest.clean();
@@ -81,9 +78,5 @@ public class ITCombinationTest extends ITKylinQueryTest {
ITKylinQueryTest.setupAll();
RemoveBlackoutRealizationsRule.blackList.clear();
- if (excludeViewCubes) {
- RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 3a7b685..227fa0a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -62,12 +62,9 @@ public class ITKylinQueryTest extends KylinTestBase {
priorities.put(RealizationType.INVERTED_INDEX, 0);
Candidate.setPriorities(priorities);
- joinType = "inner";
+ joinType = "left";
setupAll();
-
- RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
@AfterClass
@@ -152,11 +149,11 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query02.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_limit/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
- runSQL(sqlFile, true, true);
+ //runSQL(sqlFile, true, true);
runSQL(sqlFile, true, false);
}
}
@@ -324,18 +321,10 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testLimitEnabled() throws Exception {
- try {
- //other cubes have strange aggregation groups
- RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]");
-
- List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
- for (File sqlFile : sqlFiles) {
- runSQL(sqlFile, false, false);
- assertTrue(checkFinalPushDownLimit());
- }
-
- } finally {
- RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]");
+ List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
+ for (File sqlFile : sqlFiles) {
+ runSQL(sqlFile, false, false);
+ assertTrue(checkFinalPushDownLimit());
}
}
@@ -381,7 +370,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSelectStarColumnCount() throws Exception {
- execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
- execAndCompColumnCount("select * from test_kylin_fact", 9);
+ execAndCompColumnCount("select * from test_kylin_fact limit 10", 12);
+ execAndCompColumnCount("select * from test_kylin_fact", 12);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_derived/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql
index 959a59c..f1f331c 100755
--- a/kylin-it/src/test/resources/query/sql_derived/query12.sql
+++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql
@@ -18,7 +18,7 @@
select UPD_USER,count(1) as CNT
from TEST_KYLIN_FACT as TEST_KYLIN_FACT
-JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+inner join TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
where UPD_USER not in ('USER_Y')
group by UPD_USER
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
index 0c163a6..762ea38 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
@@ -19,6 +19,6 @@
select lstg_format_name, cal_dt,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct user_id) as user_count
+ count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count
from test_kylin_fact
group by lstg_format_name, cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
index 62142a2..b05f022 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct user_id) as user_count
+ count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
index 88ce532..11418d1 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct user_id) as user_count
+ count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
index 7f739e7..2be8963 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
@@ -18,7 +18,7 @@
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
, count(1) as TRANS_CNT
- , count(distinct user_id) as user_count
+ , count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count
, count(distinct site_name) as site_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
index 13de5a7..7306824 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
@@ -18,7 +18,7 @@
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
, count(1) as TRANS_CNT
- , count(distinct user_id) as user_count
+ , count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count
, count(distinct site_name) as site_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
index 15e274a..62d34da 100644
--- a/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
@@ -18,13 +18,13 @@
select
week_beg_dt as week,
-intersect_count( seller_id, lstg_format_name, array['FP-GTC']) as a,
-intersect_count( seller_id, lstg_format_name, array['Auction']) as b,
-intersect_count( seller_id, lstg_format_name, array['Others']) as c,
-intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction']) as ab,
-intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Others']) as ac,
-intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction', 'Others']) as abc,
-count(distinct seller_id) as sellers,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC']) as a,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['Auction']) as b,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['Others']) as c,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Auction']) as ab,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Others']) as ac,
+intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Auction', 'Others']) as abc,
+count(distinct TEST_COUNT_DISTINCT_BITMAP) as sellers,
count(*) as cnt
from test_kylin_fact left join edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DT
where week_beg_dt in (DATE '2013-12-22', DATE '2012-06-23')
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_limit/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_limit/query01.sql b/kylin-it/src/test/resources/query/sql_limit/query01.sql
index fca8175..0752165 100644
--- a/kylin-it/src/test/resources/query/sql_limit/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_limit/query01.sql
@@ -16,6 +16,10 @@
-- limitations under the License.
--
-select * from test_kylin_fact
- where lstg_format_name='FP-GTC'
- limit 20
+select
+ *
+from test_kylin_fact
+ inner join TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+where lstg_format_name='FP-GTC'
+limit 20
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query04.sql b/kylin-it/src/test/resources/query/sql_raw/query04.sql
index 2dc7d25..22f6ac2 100644
--- a/kylin-it/src/test/resources/query/sql_raw/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_raw/query04.sql
@@ -16,4 +16,4 @@
-- limitations under the License.
--
-select CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN'
+select CAL_DT,LSTG_FORMAT_NAME,PRICE,TRANS_ID from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query06.sql b/kylin-it/src/test/resources/query/sql_raw/query06.sql
index 0701c1f..7080ab5 100644
--- a/kylin-it/src/test/resources/query/sql_raw/query06.sql
+++ b/kylin-it/src/test/resources/query/sql_raw/query06.sql
@@ -16,4 +16,4 @@
-- limitations under the License.
--
-select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,CAL_DT,LEAF_CATEG_ID,PRICE from test_kylin_fact
\ No newline at end of file
+select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,CAL_DT,LEAF_CATEG_ID,PRICE,TRANS_ID from test_kylin_fact
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query23.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query23.sql b/kylin-it/src/test/resources/query/sql_raw/query23.sql
index 89d61db..5be4ad0 100644
--- a/kylin-it/src/test/resources/query/sql_raw/query23.sql
+++ b/kylin-it/src/test/resources/query/sql_raw/query23.sql
@@ -16,7 +16,7 @@
-- limitations under the License.
--
-select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+select test_kylin_fact.CAL_DT,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query24.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query24.sql b/kylin-it/src/test/resources/query/sql_raw/query24.sql
deleted file mode 100644
index 46c7329..0000000
--- a/kylin-it/src/test/resources/query/sql_raw/query24.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'