You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by su...@apache.org on 2016/05/22 02:33:12 UTC
[2/6] kylin git commit: KYLIN-1379 More stable and functional precise
count distinct implements after KYLIN-1186
KYLIN-1379 More stable and functional precise count distinct implements after KYLIN-1186
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/291a455d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/291a455d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/291a455d
Branch: refs/heads/KYLIN-1379-1705-1718-1719
Commit: 291a455d4e1527c9deea04f1c835e9369f9192cd
Parents: 1587b29
Author: sunyerui <su...@gmail.com>
Authored: Fri May 20 19:14:24 2016 +0800
Committer: sunyerui <su...@gmail.com>
Committed: Sun May 22 10:30:09 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapMeasureType.java | 51 ++++++++++++++++----
.../kylin/metadata/model/FunctionDesc.java | 10 +++-
...t_kylin_cube_without_slr_left_join_desc.json | 17 ++-----
.../query/sql_distinct_precisely/query00.sql | 2 +-
.../query/sql_distinct_precisely/query01.sql | 2 +-
.../query/sql_distinct_precisely/query02.sql | 2 +-
.../query/sql_distinct_precisely/query03.sql | 2 +-
.../query/sql_distinct_precisely/query04.sql | 2 +-
.../query/sql_distinct_precisely/query05.sql | 2 +-
.../query/sql_distinct_precisely/query06.sql | 2 +-
.../query/sql_distinct_precisely/query07.sql | 2 +-
webapp/app/js/model/cubeConfig.js | 2 +-
12 files changed, 61 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index def3aee..da7b405 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -29,6 +29,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -77,14 +78,8 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
throw new IllegalArgumentException("BitmapMeasureType datatype is not " + DATATYPE_BITMAP + " but " + functionDesc.getReturnDataType().getName());
List<TblColRef> colRefs = functionDesc.getParameter().getColRefs();
- if (colRefs.size() != 1) {
- throw new IllegalArgumentException("BitmapMeasureType col parameters count is not 1 but " + colRefs.size());
- }
-
- TblColRef colRef = colRefs.get(0);
- DataType type = colRef.getType();
- if (!type.isIntegerFamily()) {
- throw new IllegalArgumentException("BitmapMeasureType col type is not IntegerFamily but " + type.getName() + " of column " + colRef.getCanonicalName());
+ if (colRefs.size() != 1 && colRefs.size() != 2) {
+ throw new IllegalArgumentException("Bitmap measure need 1 or 2 parameters, but has " + colRefs.size());
}
}
@@ -100,10 +95,23 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
@Override
public BitmapCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ List<TblColRef> literalCols = measureDesc.getFunction().getParameter().getColRefs();
+ TblColRef literalCol = null;
+ if (literalCols.size() == 1) {
+ literalCol = literalCols.get(0);
+ } else if (literalCols.size() == 2) {
+ literalCol = literalCols.get(1);
+ } else {
+ throw new IllegalArgumentException("Bitmap measure need 1 or 2 parameters");
+ }
+ Dictionary<String> dictionary = dictionaryMap.get(literalCol);
BitmapCounter bitmap = current;
bitmap.clear();
- for (String v : values)
- bitmap.add(v);
+ // bitmap measure may have two values due to two parameters, only the first value should be ingested
+ if (values != null && values.length > 0 && values[0] != null) {
+ int id = dictionary.getIdFromValue(values[0]);
+ bitmap.add(id);
+ }
return bitmap;
}
};
@@ -114,12 +122,35 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
return new BitmapAggregator();
}
+ /**
+ * generate dict with first col by default, and with second col if specified
+ *
+ * Typical case: we have col uuid, and another col flag_uuid (if flag==1, uuid, null),
+ * the metrics count(distinct uuid) and count(distinct flag_uuid) should both generate dict with uuid, instead of uuid and flag_uuid
+ */
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ List<TblColRef> literalCols = functionDesc.getParameter().getColRefs();
+ if (literalCols.size() == 1) {
+ return Collections.singletonList(literalCols.get(0));
+ } else if (literalCols.size() == 2) {
+ return Collections.singletonList(literalCols.get(1));
+ } else {
+ throw new IllegalArgumentException("Bitmap measure need 1 or 2 parameters");
+ }
+ }
+
@Override
public boolean needRewrite() {
return true;
}
@Override
+ public boolean needCubeLevelDictionary() {
+ return true;
+ }
+
+ @Override
public Class<?> getRewriteCalciteAggrFunctionClass() {
return BitmapDistinctCountAggFunc.class;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index c85f0e8..e1a9e88 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -293,8 +293,14 @@ public class FunctionDesc {
if (parameter == null) {
if (other.parameter != null)
return false;
- } else if (!parameter.equals(other.parameter))
- return false;
+ } else {
+ if (isCountDistinct()
+ && (parameter.getType() == null ? other.parameter.getType() == null : parameter.getType().equals(other.parameter.getType()))
+ && (parameter.getValue() == null ? other.parameter.getType() == null : parameter.getValue().equals(other.parameter.getValue())))
+ return true;
+ else if (!parameter.equals(other.parameter))
+ return false;
+ }
}
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 0ba85d9..e835e06 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -104,7 +104,7 @@
},
"dependent_measure_ref" : null
}, {
- "name" : "SELLER_CNT_HLL",
+ "name" : "SELLER_CNT_BITMAP",
"function" : {
"expression" : "COUNT_DISTINCT",
"parameter" : {
@@ -112,7 +112,7 @@
"value" : "SELLER_ID",
"next_parameter" : null
},
- "returntype" : "hllc(10)"
+ "returntype" : "bitmap"
},
"dependent_measure_ref" : null
}, {
@@ -132,17 +132,6 @@
},
"dependent_measure_ref" : null
}, {
- "name": "LEAF_CATEG_ID_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LEAF_CATEG_ID"
- },
- "returntype": "bitmap"
- },
- "dependent_measure_ref": null
- }, {
"name" : "TOP_SELLER",
"function" : {
"expression" : "TOP_N",
@@ -249,7 +238,7 @@
"name" : "f2",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "seller_cnt_hll", "seller_format_cnt", "leaf_categ_id_bitmap" ]
+ "measure_refs" : [ "seller_cnt_bitmap", "seller_format_cnt"]
} ]
}, {
"name" : "f3",
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
index e1e4a9e..a3948c3 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
@@ -19,6 +19,6 @@
select lstg_format_name, cal_dt,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
group by lstg_format_name, cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
index c1868b8..e8579ef 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
index 5a3527a..48f49e9 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
index dacdc87..3bf72f1 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
@@ -17,7 +17,7 @@
--
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct test_kylin_fact.leaf_categ_id) as LEAF_CATEG_CNT
+ , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
index ff511c3..b9fcff4 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
@@ -17,7 +17,7 @@
--
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct test_kylin_fact.leaf_categ_id) as LEAF_CATEG_CNT
+ , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
index 3d5e5e8..dea09f7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
group by lstg_format_name
order by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
index 858c92e..eb12620 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
index 41252c4..9bd2663 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
@@ -19,6 +19,6 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ count(distinct seller_id) as seller_count
from test_kylin_fact
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/291a455d/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 784b081..962c65d 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -47,7 +47,7 @@ KylinApp.constant('cubeConfig', {
{name: 'Error Rate < 2.44%', value: 'hllc14'},
{name: 'Error Rate < 1.72%', value: 'hllc15'},
{name: 'Error Rate < 1.22%', value: 'hllc16'},
- {name: 'Precisely (Only for Integer Family column)', value: 'bitmap'}
+ {name: 'Precisely (More Memory And Storage Needed)', value: 'bitmap'}
],
topNTypes: [
{name: 'Top 10', value: "topn(10)"},