You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/05/25 05:39:21 UTC
[kylin] branch master updated (c3b1f0c -> 7437c1a)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from c3b1f0c KYLIN-4469 Fix clone model error
add d896fce KYLIN-4281 Precisely set the data type of tuple expression
add 5fd7249 KYLIN-4275 Fix result of count(case when) is not correct
add 94b707f KYLIN-4282 support case when in count (distinct)
add b0fadb0 KYLIN-4282 support case when in count (distinct) if all related columns are dims
add 873d42c KYLIN-3361 Add a two layer udaf stddev_sum
add 7437c1a KYLIN-3487 add a bitmap map-based new measure for precise count distinct
No new revisions were added by this update.
Summary of changes:
.../apache/kylin/cube/CubeCapabilityChecker.java | 12 +-
.../org/apache/kylin/cube/CubeDescManager.java | 3 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 5 +
.../cube/gridtable/CuboidToGridTableMapping.java | 4 +-
.../gridtable/CuboidToGridTableMappingExt.java | 11 -
.../kylin/cube/gridtable/ScanRangePlannerBase.java | 14 +-
.../java/org/apache/kylin/cube/model/CubeDesc.java | 37 +++
.../apache/kylin/cube/util/KeyValueBuilder.java | 37 +++
.../gridtable/GTAggregateTransformScanner.java | 300 +++++++++++++++++++++
.../apache/kylin/gridtable/GTFunctionScanner.java | 3 +-
.../java/org/apache/kylin/gridtable/GTRecord.java | 4 -
.../org/apache/kylin/gridtable/GTScanRequest.java | 29 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 19 +-
.../kylin/gridtable/GTTwoLayerAggregateParam.java | 96 +++++++
.../gridtable/GTTwoLayerAggregateScanner.java | 63 +++++
.../java/org/apache/kylin/gridtable/GTUtil.java | 6 +-
.../java/org/apache/kylin/cube/CubeDescTest.java | 6 +
.../gridtable/GTTwoLayerAggregateParamTest.java | 59 ++++
.../kylin/measure/MeasureTransformation.java | 9 +-
.../java/org/apache/kylin/measure/MeasureType.java | 5 +
.../apache/kylin/measure/MeasureTypeFactory.java | 4 +
.../kylin/measure/bitmap/BitmapSerializer.java | 3 +
.../apache/kylin/measure/hllc/HLLCSerializer.java | 3 +
.../apache/kylin/measure/map/MapKeySerializer.java | 22 +-
.../kylin/measure/map/bitmap/BitmapCounterMap.java | 116 ++++++++
.../bitmap/BitmapCounterMapFactory.java} | 13 +-
.../bitmap/BitmapMapAggregator.java} | 17 +-
.../bitmap/BitmapMapDistinctCountAggFunc.java} | 22 +-
.../bitmap/BitmapMapMeasureType.java} | 105 ++++----
.../measure/map/bitmap/BitmapMapSerializer.java | 77 ++++++
.../map/bitmap/RoaringBitmapCounterMap.java | 23 +-
.../bitmap/RoaringBitmapCounterMapFactory.java} | 29 +-
.../map/bitmap/SegmentStartTimeKeySerializer.java} | 46 ++--
.../StandardDeviationAggFunc.java} | 36 +--
.../apache/kylin/measure/stddev/StdDevCounter.java | 89 ++++++
.../kylin/measure/stddev/StdDevSerializer.java | 46 ++--
.../StdDevSumMeasureType.java} | 61 +++--
.../StdDevTransformation.java} | 65 ++---
.../metadata/expression/BinaryTupleExpression.java | 110 ++++++--
.../metadata/expression/CaseTupleExpression.java | 25 +-
.../metadata/expression/ColumnTupleExpression.java | 14 +-
...xpression.java => ConstantTupleExpression.java} | 60 +++--
.../expression/ExpressionColCollector.java | 11 +-
.../expression/ExpressionCountDistributor.java | 23 +-
.../metadata/expression/ExpressionVisitor.java | 16 +-
.../metadata/expression/NoneTupleExpression.java | 2 +-
.../expression/RexCallTupleExpression.java | 5 +-
.../metadata/expression/StringTupleExpression.java | 89 ------
.../kylin/metadata/expression/TupleExpression.java | 77 +++++-
.../expression/TupleExpressionSerializer.java | 40 ++-
...CountDistinctExpressionDynamicFunctionDesc.java | 48 ++++
.../kylin/metadata/model/DynamicFunctionDesc.java | 52 +---
...esc.java => ExpressionDynamicFunctionDesc.java} | 43 ++-
.../apache/kylin/metadata/model/FunctionDesc.java | 18 +-
.../metadata/model/StdDevSumFunctionDesc.java | 85 ++++++
....java => SumExpressionDynamicFunctionDesc.java} | 35 ++-
...ntStatusEnum.java => TwoLayerFunctionDesc.java} | 19 +-
.../measure/map/bitmap/BitmapCounterMapTest.java | 59 ++++
.../map/bitmap/BitmapMapAggregatorTest.java | 57 ++++
.../expression/ExpressionCountDistributorTest.java | 208 +++++++++-----
.../expression/TupleExpressionSerializerTest.java | 26 +-
.../metadata/expression/TupleExpressionTest.java | 51 ++--
.../storage/gtrecord/CubeScanRangePlanner.java | 55 +++-
.../kylin/storage/gtrecord/CubeTupleConverter.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 15 ++
.../kylin/engine/spark/KylinKryoRegistrator.java | 9 +
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
.../resources/query/sql_expression/query02.sql | 2 +-
.../sql_expression/{query02.sql => query07.sql} | 8 +-
.../sql_expression/{query03.sql => query08.sql} | 8 +-
.../sql_expression/{query02.sql => query09.sql} | 16 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 214 +++++++++++----
.../apache/kylin/query/relnode/OLAPContext.java | 6 +-
.../apache/kylin/query/relnode/OLAPProjectRel.java | 14 +-
.../apache/kylin/query/relnode/OLAPUnionRel.java | 4 +-
.../relnode/visitor/TupleExpressionVisitor.java | 39 +--
.../v2/coprocessor/endpoint/CubeVisitService.java | 22 +-
.../coprocessor/endpoint/CubeVisitServiceTest.java | 287 +++++++++++++++++---
78 files changed, 2515 insertions(+), 860 deletions(-)
create mode 100644 core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateTransformScanner.java
create mode 100644 core-cube/src/main/java/org/apache/kylin/gridtable/GTTwoLayerAggregateParam.java
create mode 100644 core-cube/src/main/java/org/apache/kylin/gridtable/GTTwoLayerAggregateScanner.java
create mode 100644 core-cube/src/test/java/org/apache/kylin/gridtable/GTTwoLayerAggregateParamTest.java
copy core-common/src/main/java/org/apache/kylin/common/Closeable.java => core-metadata/src/main/java/org/apache/kylin/measure/MeasureTransformation.java (86%)
copy core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java => core-metadata/src/main/java/org/apache/kylin/measure/map/MapKeySerializer.java (78%)
create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/map/bitmap/BitmapCounterMap.java
copy core-metadata/src/main/java/org/apache/kylin/measure/{bitmap/BitmapCounterFactory.java => map/bitmap/BitmapCounterMapFactory.java} (73%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{bitmap/BitmapAggregator.java => map/bitmap/BitmapMapAggregator.java} (76%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{bitmap/BitmapDistinctCountAggFunc.java => map/bitmap/BitmapMapDistinctCountAggFunc.java} (62%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{bitmap/BitmapMeasureType.java => map/bitmap/BitmapMapMeasureType.java} (52%)
create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/map/bitmap/BitmapMapSerializer.java
copy core-job/src/test/java/org/apache/kylin/job/NoErrorStatusExecutable.java => core-metadata/src/main/java/org/apache/kylin/measure/map/bitmap/RoaringBitmapCounterMap.java (62%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{bitmap/RoaringBitmapCounterFactory.java => map/bitmap/RoaringBitmapCounterMapFactory.java} (55%)
copy core-metadata/src/main/java/org/apache/kylin/{metadata/datatype/Long8Serializer.java => measure/map/bitmap/SegmentStartTimeKeySerializer.java} (58%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{percentile/PercentileAggFunc.java => stddev/StandardDeviationAggFunc.java} (57%)
create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/stddev/StdDevCounter.java
copy core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimensionSerializer.java => core-metadata/src/main/java/org/apache/kylin/measure/stddev/StdDevSerializer.java (55%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{percentile/PercentileMeasureType.java => stddev/StdDevSumMeasureType.java} (54%)
copy core-metadata/src/main/java/org/apache/kylin/measure/{hllc/HLLCAggregator.java => stddev/StdDevTransformation.java} (50%)
rename core-metadata/src/main/java/org/apache/kylin/metadata/expression/{NumberTupleExpression.java => ConstantTupleExpression.java} (58%)
delete mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/expression/StringTupleExpression.java
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/model/CountDistinctExpressionDynamicFunctionDesc.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/{DynamicFunctionDesc.java => ExpressionDynamicFunctionDesc.java} (68%)
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/model/StdDevSumFunctionDesc.java
rename core-metadata/src/main/java/org/apache/kylin/metadata/model/{SumDynamicFunctionDesc.java => SumExpressionDynamicFunctionDesc.java} (65%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/{SegmentStatusEnum.java => TwoLayerFunctionDesc.java} (78%)
create mode 100644 core-metadata/src/test/java/org/apache/kylin/measure/map/bitmap/BitmapCounterMapTest.java
create mode 100644 core-metadata/src/test/java/org/apache/kylin/measure/map/bitmap/BitmapMapAggregatorTest.java
copy kylin-it/src/test/resources/query/sql_expression/{query02.sql => query07.sql} (82%)
copy kylin-it/src/test/resources/query/sql_expression/{query03.sql => query08.sql} (83%)
copy kylin-it/src/test/resources/query/sql_expression/{query02.sql => query09.sql} (61%)