You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/09/22 11:24:19 UTC
incubator-kylin git commit: KYLIN-943 add topn measure to existing
test cubes
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-943 ba8fcd851 -> 167386858
KYLIN-943 add topn measure to existing test cubes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/16738685
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/16738685
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/16738685
Branch: refs/heads/KYLIN-943
Commit: 167386858fa251222888c5bc5d3b1eb5a2ff232d
Parents: ba8fcd8
Author: shaofengshi <sh...@apache.org>
Authored: Tue Sep 22 17:24:05 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Sep 22 17:24:05 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 15 +-----------
.../kylin/cube/CubeCapabilityChecker.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 6 +++++
.../kylin/gridtable/GTAggregateScanner.java | 3 ++-
.../test_kylin_cube_without_slr_desc.json | 25 ++++++++++++++++++++
...t_kylin_cube_without_slr_left_join_desc.json | 25 ++++++++++++++++++++
.../localmeta/project/default.json | 5 ----
examples/test_case_data/sandbox/mapred-site.xml | 18 +++++++-------
query/src/test/resources/query/sql/query81.sql | 6 ++++-
query/src/test/resources/query/sql/query82.sql | 13 ++++++----
10 files changed, 83 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 4c17630..5aa108d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -133,7 +133,7 @@ public class BuildCubeWithEngineTest {
}
private void testLeft() throws Exception {
- String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2", "testLeftJoinTopNCube" };
+ String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" };
runTestAndAssertSucceed(testCase);
}
@@ -186,19 +186,6 @@ public class BuildCubeWithEngineTest {
}
}
-
- @SuppressWarnings("unused")
- // called by reflection
- private List<String> testLeftJoinTopNCube() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2050-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_topn", date1, date2));
- return result;
- }
-
@SuppressWarnings("unused")
// called by reflection
private List<String> testInnerJoinCube2() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 628340e..df0d9c6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -79,7 +79,7 @@ public class CubeCapabilityChecker {
return true;
}
- private static boolean isMatchedWithTopN(Collection<TblColRef> dimensionColumns, CubeInstance cube, SQLDigest digest) {
+ public static boolean isMatchedWithTopN(Collection<TblColRef> dimensionColumns, CubeInstance cube, SQLDigest digest) {
CubeDesc cubeDesc = cube.getDescriptor();
List<FunctionDesc> cubeFunctions = cubeDesc.listAllFunctions();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 4bfdb18..0811936 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -363,6 +363,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
calculatedCost += COST_WEIGHT_INNER_JOIN;
}
}
+
+ if (CubeCapabilityChecker.isMatchedWithTopN(CubeDimensionDeriver.getDimensionColumns(digest), this, digest)) {
+ // this is topN query
+ calculatedCost = calculatedCost / 3;
+ }
+
return calculatedCost;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 6950e98..9050c49 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -12,6 +12,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.measure.HLLCAggregator;
import org.apache.kylin.metadata.measure.LDCAggregator;
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.metadata.measure.TopNAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -166,7 +167,7 @@ public class GTAggregateScanner implements IGTScanner {
// skip expensive aggregation
for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator)
+ if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator || totalSum[i] instanceof TopNAggregator )
totalSum[i] = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 76675f5..0069d42 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -162,6 +162,20 @@
"returntype": "hllc(10)"
},
"dependent_measure_ref": null
+ },
+ {
+ "id": 8,
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "displaycolumn": "seller_id"
+ },
+ "returntype": "topn(100)"
+ },
+ "dependent_measure_ref": null
}
],
"rowkey": {
@@ -261,6 +275,17 @@
]
}
]
+ },
+ {
+ "name": "f3",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "top_seller"
+ ]
+ }
+ ]
}
]
},
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index d3be1c9..4cc9f5f 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -162,6 +162,20 @@
"returntype": "hllc(10)"
},
"dependent_measure_ref": null
+ },
+ {
+ "id": 8,
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "displaycolumn": "seller_id"
+ },
+ "returntype": "topn(100)"
+ },
+ "dependent_measure_ref": null
}
],
"rowkey": {
@@ -261,6 +275,17 @@
]
}
]
+ },
+ {
+ "name": "f3",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "top_seller"
+ ]
+ }
+ ]
}
]
},
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 71eb1fa..e4118ca 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -23,11 +23,6 @@
"realization": "test_kylin_cube_without_slr_left_join_empty"
},
{
- "name": "test_kylin_cube_topn",
- "type": "CUBE",
- "realization": "test_kylin_cube_topn"
- },
- {
"name": "test_kylin_ii_left_join",
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii_left_join"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 611bf8b..cba5aaa 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -3,7 +3,7 @@
<property>
<name>io.sort.mb</name>
- <value>64</value>
+ <value>128</value>
</property>
<property>
@@ -13,12 +13,12 @@
<property>
<name>mapred.job.map.memory.mb</name>
- <value>250</value>
+ <value>512</value>
</property>
<property>
<name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
+ <value>512</value>
</property>
<property>
@@ -98,7 +98,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx200m</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -108,7 +108,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>250</value>
+ <value>512</value>
</property>
<property>
@@ -153,7 +153,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>250</value>
+ <value>512</value>
</property>
<property>
@@ -203,7 +203,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>64</value>
+ <value>128</value>
</property>
<property>
@@ -218,7 +218,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx200m</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -228,7 +228,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>250</value>
+ <value>512</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/query/src/test/resources/query/sql/query81.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query81.sql b/query/src/test/resources/query/sql/query81.sql
index 39a14a5..d7cc036 100644
--- a/query/src/test/resources/query/sql/query81.sql
+++ b/query/src/test/resources/query/sql/query81.sql
@@ -18,8 +18,12 @@
SELECT
seller_id
- FROM test_kylin_fact left join edw.test_cal_dt as test_cal_dt
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
where test_kylin_fact.cal_dt < DATE '2013-02-01'
group by
test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 100
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/query/src/test/resources/query/sql/query82.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql b/query/src/test/resources/query/sql/query82.sql
index 29d67b8..82c64d6 100644
--- a/query/src/test/resources/query/sql/query82.sql
+++ b/query/src/test/resources/query/sql/query82.sql
@@ -17,9 +17,14 @@
--
SELECT
- test_kylin_fact.cal_dt, seller_id
- FROM test_kylin_fact
-left JOIN edw.test_cal_dt as test_cal_dt
+ cal_dt, seller_id from (select
+ test_kylin_fact.cal_dt, seller_id, sum(test_kylin_fact.price) as gmv
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
group by
- test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 100
+ test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by gmv desc limit 100
+ ) as abc