You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/04/09 09:44:05 UTC
[kylin] branch kylin-on-parquet updated (011ff08 -> 768d7ca)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch kylin-on-parquet
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 011ff08 KYLIN-3726: KylinSession should load spark properties from spark-defaults.conf
new 8a0d3f6 KYLIN-3777 Remove dim dict from parquet file
new 8f3195e KYLIN-3777 Remove dict from query side
new 05e4bd2 KYLIN-3777 Pass CI
new 768d7ca KYLIN-3777 Fix missing statistics resources
The 7546 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../org/apache/kylin/common/util/TimeUtil.java | 53 +++
.../java/org/apache/kylin/gridtable/GTUtil.java | 136 +-------
.../filter/BuiltInFunctionTupleFilter.java | 13 +
.../kylin/metadata/filter/CaseTupleFilter.java | 8 +
.../kylin/metadata/filter/ColumnTupleFilter.java | 10 +
.../kylin/metadata/filter/CompareTupleFilter.java | 20 ++
.../kylin/metadata/filter/ConstantTupleFilter.java | 10 +
.../kylin/metadata/filter/DynamicTupleFilter.java | 10 +
.../kylin/metadata/filter/ExtractTupleFilter.java | 10 +
.../kylin/metadata/filter/FunctionTupleFilter.java | 5 +
.../kylin/metadata/filter/LogicalTupleFilter.java | 10 +
.../apache/kylin/metadata/filter/TupleFilter.java | 17 +
.../kylin/metadata/filter/TupleFilterVisitor.java | 22 ++
.../kylin/metadata/filter/TupleFilterVisitor2.java | 42 +++
.../filter/TupleFilterVisitor2Adaptor.java | 100 ++++++
.../apache/kylin/metadata/filter/TupleFilters.java | 146 ++++++++
.../metadata/filter/UDF/MassInTupleFilter.java | 5 +
.../metadata/filter/UnsupportedTupleFilter.java | 10 +
.../org/apache/kylin/storage/IStorageQuery.java | 4 +
.../org/apache/kylin/storage/StorageContext.java | 8 +
.../storage/gtrecord/CubeScanRangePlanner.java | 10 -
.../kylin/storage/hybrid/HybridStorageQuery.java | 12 +
.../kylin/engine/mr/common/AbstractHadoopJob.java | 1 +
kylin-it/jacoco-it.exec | Bin 2432563 -> 0 bytes
.../org/apache/kylin/jdbc/ITJDBCDriver2Test.java | 9 +
.../apache/kylin/query/ITFailfastQuery2Test.java | 6 +
.../org/apache/kylin/query/ITKylinQuery2Test.java | 81 ++++-
.../org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 6 +-
.../apache/kylin/query/relnode/OLAPFilterRel.java | 21 +-
.../apache/kylin/query/relnode/OLAPProjectRel.java | 12 +-
.../query/relnode/visitor/TupleFilterVisitor.java | 1 +
.../kylin/rest/service/KylinQueryTimeoutTest.java | 5 +
.../storage/hbase/cube/v2/CubeStorageQuery.java | 5 +
.../apache/kylin/storage/parquet/NameMapping.java | 15 +-
.../kylin/storage/parquet/NameMappingFactory.java | 111 ++++++
.../kylin/storage/parquet/ParquetSchema.java | 151 ++++++++
.../kylin/storage/parquet/ParquetStorage.java | 4 +-
.../kylin/storage/parquet/cube/ColumnFiller.java | 7 +-
.../kylin/storage/parquet/cube/CubeSparkRPC.java | 106 ------
.../storage/parquet/cube/DerivedColumnFiller.java | 56 +++
.../storage/parquet/cube/DerivedIndexMapping.java | 91 +++++
...CubeStorageQuery.java => LookupTableCache.java} | 31 +-
.../storage/parquet/cube/ParquetColumnFiller.java | 94 +++++
.../storage/parquet/cube/ParquetStorageQuery.java | 381 +++++++++++++++++++++
.../storage/parquet/cube/ParquetTupleIterator.java | 162 +++++++++
.../{CubeStorageQuery.java => PkColumnFiller.java} | 20 +-
.../storage/parquet/cube/ToSparkFilterVisitor.java | 301 ++++++++++++++++
.../parquet/cube/TranslateDerivedVisitor.java | 334 ++++++++++++++++++
.../parquet/cube/UnsupportedFilterException.java | 15 +-
.../storage/parquet/spark/ParquetPayload.java | 220 ------------
.../kylin/storage/parquet/spark/ParquetTask.java | 309 -----------------
.../storage/parquet/spark/SparkSubmitter.java | 37 --
.../spark/gtscanner/ParquetRecordGTScanner.java | 139 --------
.../gtscanner/ParquetRecordGTScanner4Cube.java | 32 --
.../parquet/steps/ConvertToParquetReducer.java | 22 +-
.../storage/parquet/steps/MRCubeParquetJob.java | 21 +-
.../storage/parquet/steps/ParquetConvertor.java | 238 ++++---------
.../storage/parquet/steps/SparkCubeParquet.java | 46 ++-
60 files changed, 2505 insertions(+), 1254 deletions(-)
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor.java
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor2.java
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor2Adaptor.java
create mode 100644 core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilters.java
delete mode 100644 kylin-it/jacoco-it.exec
copy core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java => storage-parquet/src/main/java/org/apache/kylin/storage/parquet/NameMapping.java (78%)
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/NameMappingFactory.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/ParquetSchema.java
copy jdbc/src/main/java/org/apache/kylin/jdbc/JdbcFactory.java => storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ColumnFiller.java (82%)
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/CubeSparkRPC.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/DerivedColumnFiller.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/DerivedIndexMapping.java
copy storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/{CubeStorageQuery.java => LookupTableCache.java} (50%)
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetColumnFiller.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetStorageQuery.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetTupleIterator.java
rename storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/{CubeStorageQuery.java => PkColumnFiller.java} (66%)
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ToSparkFilterVisitor.java
create mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/TranslateDerivedVisitor.java
copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java => storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/UnsupportedFilterException.java (77%)
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/ParquetPayload.java
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/ParquetTask.java
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/SparkSubmitter.java
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/gtscanner/ParquetRecordGTScanner.java
delete mode 100644 storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/gtscanner/ParquetRecordGTScanner4Cube.java