You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by zh...@apache.org on 2022/05/15 06:11:00 UTC
[kylin] branch kylin-soft-affinity-local-cache updated (c03de129d7 -> 65bd536741)
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a change to branch kylin-soft-affinity-local-cache
in repository https://gitbox.apache.org/repos/asf/kylin.git
discard c03de129d7 Upgrade alluxio client verion to 2.7.4
discard c06cfa4841 Pre-init KylinCacheFileSystem to fix s3a issue
discard d31e8f5050 Support soft affinity and local cache feature
add 83f8a3d450 minor, fix invalid config
add 626c971cb1 remove useless codes
add 15825b3b59 minor, update pull request template
add f6f3ae5808 minor, remove sonarcloud link as it stops java8 support
new 0f5b404a68 Support soft affinity and local cache feature
new b893faf8bd Pre-init KylinCacheFileSystem to fix s3a issue
new 65bd536741 Upgrade alluxio client verion to 2.7.4
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c03de129d7)
\
N -- N -- N refs/heads/kylin-soft-affinity-local-cache (65bd536741)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/pull_request_template.md | 6 +-
README.md | 7 +-
.../org/apache/kylin/common/KylinConfigBase.java | 661 +-
.../org/apache/kylin/common/KylinConfigExt.java | 11 +-
.../src/main/resources/kylin-defaults.properties | 2 +-
.../.settings/org.eclipse.core.resources.prefs | 6 -
.../.settings/org.eclipse.jdt.core.prefs | 386 -
core-dictionary/.settings/org.eclipse.jdt.ui.prefs | 7 -
core-dictionary/DEPRECATED_MODULE | 19 -
.../apache/kylin/dict/AppendTrieDictionary.java | 290 -
.../kylin/dict/BuiltInFunctionTransformer.java | 197 -
.../java/org/apache/kylin/dict/ByteComparator.java | 44 -
.../java/org/apache/kylin/dict/BytesConverter.java | 29 -
.../org/apache/kylin/dict/CacheDictionary.java | 129 -
.../org/apache/kylin/dict/DateStrDictionary.java | 155 -
.../java/org/apache/kylin/dict/DictCodeSystem.java | 69 -
.../org/apache/kylin/dict/DictionaryGenerator.java | 235 -
.../java/org/apache/kylin/dict/DictionaryInfo.java | 167 -
.../kylin/dict/DictionaryInfoSerializer.java | 86 -
.../org/apache/kylin/dict/DictionaryManager.java | 453 -
.../org/apache/kylin/dict/DictionaryProvider.java | 30 -
.../apache/kylin/dict/DictionarySerializer.java | 79 -
.../kylin/dict/DistinctColumnValuesProvider.java | 35 -
.../apache/kylin/dict/GlobalDictionaryBuilder.java | 120 -
.../org/apache/kylin/dict/IDictionaryBuilder.java | 41 -
.../kylin/dict/IDictionaryValueEnumerator.java | 33 -
.../dict/IterableDictionaryValueEnumerator.java | 52 -
.../dict/MultipleDictionaryValueEnumerator.java | 88 -
.../apache/kylin/dict/Number2BytesConverter.java | 267 -
.../org/apache/kylin/dict/NumberDictionary.java | 55 -
.../org/apache/kylin/dict/NumberDictionary2.java | 46 -
.../apache/kylin/dict/NumberDictionaryBuilder.java | 42 -
.../kylin/dict/NumberDictionaryForestBuilder.java | 50 -
.../apache/kylin/dict/StringBytesConverter.java | 42 -
.../kylin/dict/TableColumnValueEnumerator.java | 74 -
.../dict/TableColumnValueSortedEnumerator.java | 154 -
.../org/apache/kylin/dict/TimeStrDictionary.java | 121 -
.../java/org/apache/kylin/dict/TrieDictionary.java | 483 -
.../apache/kylin/dict/TrieDictionaryBuilder.java | 529 -
.../apache/kylin/dict/TrieDictionaryForest.java | 400 -
.../kylin/dict/TrieDictionaryForestBuilder.java | 173 -
.../apache/kylin/dict/global/AppendDictNode.java | 384 -
.../apache/kylin/dict/global/AppendDictSlice.java | 289 -
.../kylin/dict/global/AppendDictSliceKey.java | 75 -
.../dict/global/AppendTrieDictionaryBuilder.java | 283 -
.../dict/global/AppendTrieDictionaryChecker.java | 102 -
.../kylin/dict/global/GlobalDictHDFSStore.java | 439 -
.../kylin/dict/global/GlobalDictMetadata.java | 50 -
.../apache/kylin/dict/global/GlobalDictStore.java | 103 -
.../dict/global/SegmentAppendTrieDictBuilder.java | 86 -
.../org/apache/kylin/dict/lookup/ILookupTable.java | 32 -
.../kylin/dict/lookup/LookupProviderFactory.java | 30 -
.../kylin/dict/lookup/LookupStringTable.java | 126 -
.../org/apache/kylin/dict/lookup/LookupTable.java | 181 -
.../org/apache/kylin/dict/lookup/SnapshotCLI.java | 51 -
.../apache/kylin/dict/lookup/SnapshotManager.java | 283 -
.../apache/kylin/dict/lookup/SnapshotTable.java | 312 -
.../kylin/dict/lookup/SnapshotTableSerializer.java | 79 -
.../kylin/dict/AppendTrieDictionaryTest.java | 707 -
.../apache/kylin/dict/DateStrDictionaryTest.java | 205 -
.../apache/kylin/dict/DictionaryManagerTest.java | 104 -
.../apache/kylin/dict/DictionaryProviderTest.java | 104 -
.../org/apache/kylin/dict/MockupReadableTable.java | 88 -
.../MultipleDictionaryValueEnumeratorTest.java | 217 -
.../apache/kylin/dict/NumberDictionaryTest.java | 216 -
.../apache/kylin/dict/TimeStrDictionaryTest.java | 151 -
.../kylin/dict/TrieDictionaryForestBenchmark.java | 174 -
.../kylin/dict/TrieDictionaryForestTest.java | 1079 -
.../org/apache/kylin/dict/TrieDictionaryTest.java | 561 -
.../apache/kylin/dict/lookup/LookupTableTest.java | 185 -
.../kylin/dict/lookup/SnapshotManagerTest.java | 171 -
.../resources/dict/dw_category_grouping_names.dat | 448529 ------------------
.../dict/english-words.80 (scowl-2015.05.18).txt | 139304 ------
datasource-sdk/DEPRECATED_MODULE | 19 -
.../datasource/adaptor/AbstractJdbcAdaptor.java | 448 -
.../sdk/datasource/adaptor/AdaptorConfig.java | 76 -
.../sdk/datasource/adaptor/DefaultAdaptor.java | 355 -
.../kylin/sdk/datasource/adaptor/H2Adaptor.java | 42 -
.../kylin/sdk/datasource/adaptor/MysqlAdaptor.java | 98 -
.../sdk/datasource/adaptor/PostgresqlAdaptor.java | 107 -
.../sdk/datasource/framework/AdaptorFactory.java | 28 -
.../framework/FixedCachedRowSetImpl.java | 325 -
.../sdk/datasource/framework/JdbcConnector.java | 186 -
.../framework/SourceConnectorFactory.java | 62 -
.../sdk/datasource/framework/conv/ConvMaster.java | 150 -
.../datasource/framework/conv/ConvSqlWriter.java | 231 -
.../framework/conv/DefaultConfigurer.java | 137 -
.../framework/conv/GenericSqlConverter.java | 57 -
.../datasource/framework/conv/ParamNodeParser.java | 45 -
.../datasource/framework/conv/SqlConverter.java | 120 -
.../framework/conv/SqlNodeConverter.java | 129 -
.../datasource/framework/conv/SqlParamsFinder.java | 109 -
.../datasource/framework/def/DataSourceDef.java | 219 -
.../framework/def/DataSourceDefProvider.java | 123 -
.../sdk/datasource/framework/def/FunctionDef.java | 49 -
.../sdk/datasource/framework/def/PropertyDef.java | 36 -
.../sdk/datasource/framework/def/TypeDef.java | 111 -
.../sdk/datasource/framework/def/TypeMapping.java | 45 -
.../framework/utils/ExpressionComparator.java | 165 -
.../sdk/datasource/framework/utils/XmlUtil.java | 48 -
.../src/main/resources/datasource/default.xml | 202 -
.../src/main/resources/datasource/mysql.xml | 39 -
.../src/main/resources/datasource/postgresql.xml | 78 -
.../sdk/datasource/adaptor/AdaptorConfigTest.java | 36 -
.../datasource/framework/JdbcConnectorTest.java | 98 -
.../framework/SourceConnectorFactoryTest.java | 44 -
.../framework/conv/ParamNodeParserTest.java | 38 -
.../framework/conv/PostgresqlSqlConverterTest.java | 94 -
.../framework/conv/SqlConverterTest.java | 547 -
.../framework/conv/SqlParamsFinderTest.java | 62 -
.../framework/def/DataSourceDefTest.java | 82 -
.../sdk/datasource/framework/def/TypeDefTest.java | 54 -
.../src/test/resources/datasource/default.xml | 226 -
.../test/resources/datasource/default.xml.override | 202 -
.../src/test/resources/datasource/testing.xml | 122 -
engine-flink/DEPRECATED_MODULE | 19 -
engine-flink/pom.xml | 158 -
.../engine/flink/FlinkBatchCubingEngine2.java | 68 -
.../engine/flink/FlinkBatchCubingJobBuilder2.java | 125 -
.../engine/flink/FlinkBatchMergeJobBuilder2.java | 132 -
.../kylin/engine/flink/FlinkCubingByLayer.java | 636 -
.../kylin/engine/flink/FlinkCubingMerge.java | 352 -
.../apache/kylin/engine/flink/FlinkExecutable.java | 447 -
.../kylin/engine/flink/FlinkMergingDictionary.java | 309 -
.../engine/flink/FlinkOnYarnConfigMapping.java | 83 -
.../org/apache/kylin/engine/flink/FlinkUtil.java | 133 -
.../org/apache/kylin/engine/flink/IFlinkInput.java | 60 -
.../apache/kylin/engine/flink/IFlinkOutput.java | 132 -
.../flink/util/PercentileCounterSerializer.java | 67 -
.../engine/flink/FlinkOnYarnConfigMappingTest.java | 109 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
engine-mr/.settings/org.eclipse.jdt.core.prefs | 386 -
engine-mr/.settings/org.eclipse.jdt.ui.prefs | 7 -
engine-mr/DEPRECATED_MODULE | 19 -
engine-mr/pom.xml | 119 -
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 202 -
.../kylin/engine/mr/BatchMergeJobBuilder2.java | 98 -
.../kylin/engine/mr/BatchOptimizeJobBuilder2.java | 228 -
.../mr/BatchOptimizeJobCheckpointBuilder.java | 90 -
.../apache/kylin/engine/mr/ByteArrayWritable.java | 185 -
.../java/org/apache/kylin/engine/mr/CubingJob.java | 417 -
.../org/apache/kylin/engine/mr/DFSFileTable.java | 112 -
.../apache/kylin/engine/mr/DFSFileTableReader.java | 253 -
.../java/org/apache/kylin/engine/mr/IInput.java | 47 -
.../kylin/engine/mr/ILookupMaterializer.java | 40 -
.../java/org/apache/kylin/engine/mr/IMRInput.java | 68 -
.../java/org/apache/kylin/engine/mr/IMROutput.java | 79 -
.../org/apache/kylin/engine/mr/IMROutput2.java | 143 -
.../apache/kylin/engine/mr/JobBuilderSupport.java | 451 -
.../org/apache/kylin/engine/mr/KylinMapper.java | 123 -
.../org/apache/kylin/engine/mr/KylinReducer.java | 123 -
.../kylin/engine/mr/LookupMaterializeContext.java | 85 -
.../kylin/engine/mr/LookupSnapshotBuildJob.java | 84 -
.../kylin/engine/mr/LookupSnapshotJobBuilder.java | 90 -
.../kylin/engine/mr/MRBatchCubingEngine2.java | 65 -
.../java/org/apache/kylin/engine/mr/MRUtil.java | 99 -
.../kylin/engine/mr/SortedColumnDFSFile.java | 135 -
.../kylin/engine/mr/SortedColumnDFSFileReader.java | 136 -
.../kylin/engine/mr/StreamingCubingEngine.java | 35 -
.../kylin/engine/mr/StreamingCubingJobBuilder.java | 233 -
.../kylin/engine/mr/common/AbstractHadoopJob.java | 753 -
.../kylin/engine/mr/common/BaseCuboidBuilder.java | 142 -
.../kylin/engine/mr/common/BatchConstants.java | 128 -
.../engine/mr/common/ConvergeCuboidDataUtil.java | 57 -
.../kylin/engine/mr/common/CubeJobLockUtil.java | 46 -
.../kylin/engine/mr/common/CubeStatsReader.java | 544 -
.../kylin/engine/mr/common/CubeStatsWriter.java | 102 -
.../engine/mr/common/CuboidRecommenderUtil.java | 135 -
.../engine/mr/common/CuboidSchedulerUtil.java | 60 -
.../kylin/engine/mr/common/CuboidShardUtil.java | 57 -
.../engine/mr/common/CuboidStatsReaderUtil.java | 169 -
.../mr/common/DefaultSslProtocolSocketFactory.java | 150 -
.../engine/mr/common/DefaultX509TrustManager.java | 114 -
.../engine/mr/common/DictionaryGetterUtil.java | 76 -
.../kylin/engine/mr/common/HadoopCmdOutput.java | 160 -
.../engine/mr/common/HadoopJobStatusChecker.java | 66 -
.../engine/mr/common/HadoopShellExecutable.java | 99 -
.../kylin/engine/mr/common/JobInfoConverter.java | 241 -
.../kylin/engine/mr/common/JobRelatedMetaUtil.java | 105 -
.../engine/mr/common/MapReduceExecutable.java | 541 -
.../kylin/engine/mr/common/MapReduceUtil.java | 164 -
.../kylin/engine/mr/common/NDCuboidBuilder.java | 117 -
.../mr/common/SerializableConfiguration.java | 50 -
.../engine/mr/common/StatisticsDecisionUtil.java | 134 -
.../engine/mr/exception/HadoopShellException.java | 44 -
.../engine/mr/exception/MapReduceException.java | 43 -
.../mr/exception/SegmentNotFoundException.java | 44 -
.../kylin/engine/mr/steps/BaseCuboidJob.java | 40 -
.../engine/mr/steps/BaseCuboidMapperBase.java | 106 -
.../mr/steps/CalculateStatsFromBaseCuboidJob.java | 119 -
.../steps/CalculateStatsFromBaseCuboidMapper.java | 200 -
.../CalculateStatsFromBaseCuboidPartitioner.java | 59 -
.../steps/CalculateStatsFromBaseCuboidReducer.java | 115 -
.../mr/steps/ConvergeCuboidDataPartitioner.java | 67 -
.../engine/mr/steps/ConvergeCuboidDataReducer.java | 101 -
.../kylin/engine/mr/steps/CopyDictionaryStep.java | 70 -
.../kylin/engine/mr/steps/CreateDictionaryJob.java | 127 -
.../engine/mr/steps/CubingExecutableUtil.java | 159 -
.../apache/kylin/engine/mr/steps/CuboidJob.java | 192 -
.../kylin/engine/mr/steps/CuboidReducer.java | 117 -
.../mr/steps/ExtractDictionaryFromGlobalJob.java | 107 -
.../steps/ExtractDictionaryFromGlobalMapper.java | 144 -
.../mr/steps/FactDistinctColumnPartitioner.java | 76 -
.../mr/steps/FactDistinctColumnsCombiner.java | 44 -
.../engine/mr/steps/FactDistinctColumnsJob.java | 171 -
.../engine/mr/steps/FactDistinctColumnsMapper.java | 467 -
.../mr/steps/FactDistinctColumnsMapperBase.java | 103 -
.../mr/steps/FactDistinctColumnsReducer.java | 301 -
.../steps/FactDistinctColumnsReducerMapping.java | 149 -
.../mr/steps/FilterRecommendCuboidDataJob.java | 100 -
.../mr/steps/FilterRecommendCuboidDataMapper.java | 72 -
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 53 -
.../mr/steps/InMemCuboidFromBaseCuboidJob.java | 162 -
.../mr/steps/InMemCuboidFromBaseCuboidMapper.java | 83 -
.../mr/steps/InMemCuboidFromBaseCuboidReducer.java | 23 -
.../kylin/engine/mr/steps/InMemCuboidJob.java | 139 -
.../kylin/engine/mr/steps/InMemCuboidMapper.java | 60 -
.../engine/mr/steps/InMemCuboidMapperBase.java | 221 -
.../kylin/engine/mr/steps/InMemCuboidReducer.java | 102 -
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 104 -
.../engine/mr/steps/MapContextGTRecordWriter.java | 59 -
.../kylin/engine/mr/steps/MergeCuboidJob.java | 98 -
.../kylin/engine/mr/steps/MergeCuboidMapper.java | 73 -
.../kylin/engine/mr/steps/MergeDictionaryJob.java | 234 -
.../engine/mr/steps/MergeDictionaryMapper.java | 223 -
.../engine/mr/steps/MergeDictionaryReducer.java | 46 -
.../kylin/engine/mr/steps/MergeDictionaryStep.java | 149 -
.../kylin/engine/mr/steps/MergeStatisticsStep.java | 159 -
.../mr/steps/MergeStatisticsWithOldStep.java | 155 -
.../kylin/engine/mr/steps/MetadataCleanupJob.java | 183 -
.../apache/kylin/engine/mr/steps/NDCuboidJob.java | 34 -
.../kylin/engine/mr/steps/NDCuboidMapper.java | 118 -
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 173 -
.../kylin/engine/mr/steps/SegmentReEncoder.java | 255 -
.../engine/mr/steps/SelfDefineSortableKey.java | 128 -
.../kylin/engine/mr/steps/UHCDictionaryJob.java | 156 -
.../kylin/engine/mr/steps/UHCDictionaryMapper.java | 101 -
.../engine/mr/steps/UHCDictionaryPartitioner.java | 30 -
.../engine/mr/steps/UHCDictionaryReducer.java | 114 -
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 183 -
.../steps/UpdateCubeInfoAfterCheckpointStep.java | 69 -
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 122 -
.../mr/steps/UpdateCubeInfoAfterOptimizeStep.java | 73 -
.../engine/mr/steps/UpdateDictionaryStep.java | 152 -
.../engine/mr/steps/UpdateOldCuboidShardJob.java | 102 -
.../mr/steps/UpdateOldCuboidShardMapper.java | 105 -
.../engine/mr/steps/filter/UHCDictPathFilter.java | 47 -
.../mr/steps/lookup/LookupExecutableUtil.java | 132 -
.../lookup/LookupSnapshotToMetaStoreStep.java | 83 -
.../steps/lookup/UpdateCubeAfterSnapshotStep.java | 88 -
.../kylin/engine/mr/streaming/ColumnToRowJob.java | 130 -
.../engine/mr/streaming/ColumnToRowMapper.java | 38 -
.../engine/mr/streaming/ColumnToRowReducer.java | 84 -
.../engine/mr/streaming/ColumnarFilesReader.java | 73 -
.../mr/streaming/ColumnarSplitDataInputFormat.java | 34 -
.../mr/streaming/ColumnarSplitDataReader.java | 111 -
.../mr/streaming/ColumnarSplitDictInputFormat.java | 34 -
.../mr/streaming/ColumnarSplitDictReader.java | 105 -
.../mr/streaming/ColumnarSplitInputFormat.java | 133 -
.../engine/mr/streaming/ColumnarSplitReader.java | 71 -
.../kylin/engine/mr/streaming/DictsReader.java | 68 -
.../kylin/engine/mr/streaming/MergeDictJob.java | 103 -
.../kylin/engine/mr/streaming/MergeDictMapper.java | 37 -
.../engine/mr/streaming/MergeDictReducer.java | 141 -
.../kylin/engine/mr/streaming/RowRecord.java | 40 -
.../kylin/engine/mr/streaming/RowRecordReader.java | 267 -
.../kylin/engine/mr/streaming/SaveDictStep.java | 236 -
.../kylin/engine/mr/SortedColumnReaderTest.java | 327 -
.../apache/kylin/engine/mr/TableReaderTest.java | 46 -
.../mr/common/DefaultX509TrustManagerTest.java | 49 -
.../engine/mr/common/HadoopCmdOutputTest.java | 51 -
.../mr/common/HadoopJobStatusCheckerTest.java | 66 -
.../engine/mr/common/JobInfoConverterTest.java | 233 -
.../engine/mr/common/MapReduceExecutableTest.java | 50 -
.../kylin/engine/mr/steps/CubeReducerTest.java | 205 -
.../kylin/engine/mr/steps/CubeSamplingTest.java | 128 -
.../kylin/engine/mr/steps/DictColDeduperTest.java | 65 -
.../FactDistinctColumnsReducerMappingTest.java | 89 -
.../mr/steps/FactDistinctColumnsReducerTest.java | 56 -
.../HiveToBaseCuboidMapperPerformanceTest.java | 66 -
.../mr/steps/LookupMaterializeContextTest.java | 41 -
.../kylin/engine/mr/steps/MergeCuboidJobTest.java | 88 -
.../kylin/engine/mr/steps/MockupMapContext.java | 322 -
.../kylin/engine/mr/steps/NDCuboidJobTest.java | 87 -
.../kylin/engine/mr/steps/NDCuboidMapperTest.java | 118 -
.../engine/mr/steps/NewCubeSamplingMethodTest.java | 299 -
.../mr/steps/NumberDictionaryForestTest.java | 327 -
.../engine/mr/steps/SelfDefineSortableKeyTest.java | 280 -
.../mr/steps/lookup/LookupExecutableUtilTest.java | 63 -
.../src/test/resources/data/8d_cuboid/part-r-00000 | Bin 913974 -> 0 bytes
.../test/resources/data/base_cuboid/part-r-00000 | Bin 243816 -> 0 bytes
engine-mr/src/test/resources/dict/DW_SITES | Bin 5020 -> 0 bytes
.../src/test/resources/multi_file_double/data_0 | 2006 -
.../src/test/resources/multi_file_double/data_1 | 2093 -
.../src/test/resources/multi_file_double/data_2 | 2006 -
.../src/test/resources/multi_file_double/data_3 | 1947 -
.../src/test/resources/multi_file_double/data_4 | 1948 -
.../test/resources/multi_file_empty_file/empty.txt | 0
.../resources/multi_file_empty_file/has_value.txt | 2 -
engine-mr/src/test/resources/multi_file_int/data_0 | 1987 -
engine-mr/src/test/resources/multi_file_int/data_1 | 1989 -
engine-mr/src/test/resources/multi_file_int/data_2 | 1998 -
engine-mr/src/test/resources/multi_file_int/data_3 | 1996 -
engine-mr/src/test/resources/multi_file_int/data_4 | 2030 -
engine-mr/src/test/resources/multi_file_str/data_0 | 2029 -
engine-mr/src/test/resources/multi_file_str/data_1 | 1990 -
engine-mr/src/test/resources/multi_file_str/data_2 | 1992 -
engine-mr/src/test/resources/multi_file_str/data_3 | 1948 -
engine-mr/src/test/resources/multi_file_str/data_4 | 2041 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
engine-spark/.settings/org.eclipse.jdt.core.prefs | 386 -
engine-spark/.settings/org.eclipse.jdt.ui.prefs | 7 -
engine-spark/DEPRECATED_MODULE | 19 -
engine-spark/pom.xml | 149 -
.../org/apache/kylin/engine/spark/ISparkInput.java | 44 -
.../apache/kylin/engine/spark/ISparkOutput.java | 129 -
.../kylin/engine/spark/KylinKryoRegistrator.java | 318 -
.../kylin/engine/spark/KylinSparkJobListener.java | 55 -
.../kylin/engine/spark/MultipleOutputsRDD.scala | 114 -
.../engine/spark/SparkBatchCubingEngine2.java | 68 -
.../engine/spark/SparkBatchCubingJobBuilder2.java | 213 -
.../engine/spark/SparkBatchMergeJobBuilder2.java | 129 -
.../kylin/engine/spark/SparkBuildDictionary.java | 500 -
.../kylin/engine/spark/SparkColumnCardinality.java | 159 -
.../kylin/engine/spark/SparkCreatingFlatTable.java | 43 -
.../kylin/engine/spark/SparkCubingByLayer.java | 467 -
.../kylin/engine/spark/SparkCubingMerge.java | 286 -
.../apache/kylin/engine/spark/SparkExecutable.java | 553 -
.../kylin/engine/spark/SparkExecutableFactory.java | 33 -
.../kylin/engine/spark/SparkExecutableLivy.java | 280 -
.../kylin/engine/spark/SparkFactDistinct.java | 943 -
.../kylin/engine/spark/SparkMergingDictionary.java | 326 -
.../apache/kylin/engine/spark/SparkSqlBatch.java | 131 -
.../kylin/engine/spark/SparkSqlOnLivyBatch.scala | 52 -
.../kylin/engine/spark/SparkUHCDictionary.java | 321 -
.../org/apache/kylin/engine/spark/SparkUtil.java | 191 -
.../engine/spark/exception/SparkException.java | 39 -
.../kylin/engine/spark/util/IteratorUtils.java | 84 -
.../spark/util/PercentileCounterSerializer.java | 64 -
.../kylin/engine/spark/util/IteratorUtilsTest.java | 88 -
.../engine/spark/util/KyroMappingGenerator.java | 56 -
.../spark/util/PercentileSerializerTest.java | 67 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
kylin-it/.settings/org.eclipse.jdt.core.prefs | 386 -
kylin-it/.settings/org.eclipse.jdt.ui.prefs | 7 -
kylin-it/DEPRECATED_MODULE | 21 -
kylin-it/pom.xml | 576 -
.../apache/kylin/cube/ITDictionaryManagerTest.java | 136 -
.../cube/cuboid/algorithm/ITAlgorithmTestBase.java | 305 -
.../cuboid/algorithm/ITGeneticAlgorithmTest.java | 97 -
.../cuboid/algorithm/ITGreedyAlgorithmTest.java | 58 -
.../inmemcubing/ITDoggedCubeBuilderStressTest.java | 118 -
.../cube/inmemcubing/ITDoggedCubeBuilderTest.java | 220 -
.../cube/inmemcubing/ITInMemCubeBuilderTest.java | 298 -
.../kylin/dict/ITGlobalDictionaryBuilderTest.java | 165 -
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 350 -
.../kylin/job/BaseTestDistributedScheduler.java | 184 -
.../kylin/job/ITDistributedSchedulerBaseTest.java | 57 -
.../job/ITDistributedSchedulerTakeOverTest.java | 57 -
.../kylin/provision/BuildCubeWithEngine.java | 617 -
.../kylin/provision/BuildCubeWithStream.java | 435 -
.../java/org/apache/kylin/provision/MockKafka.java | 189 -
.../org/apache/kylin/provision/NetworkUtils.java | 52 -
.../kylin/provision/ProvisionWithoutBuild.java | 29 -
.../apache/kylin/query/CompareQueryBySuffix.java | 37 -
.../java/org/apache/kylin/query/H2Database.java | 215 -
.../org/apache/kylin/query/HackedDbUnitAssert.java | 259 -
.../kylin/query/ICompareQueryTranslator.java | 29 -
.../org/apache/kylin/query/ITCombinationTest.java | 85 -
.../apache/kylin/query/ITFailfastQueryTest.java | 158 -
.../org/apache/kylin/query/ITKylinQueryTest.java | 488 -
.../org/apache/kylin/query/ITMassInQueryTest.java | 160 -
.../java/org/apache/kylin/query/KylinTestBase.java | 833 -
.../kylin/realtime/BuildCubeWithStreamV2.java | 476 -
.../apache/kylin/restclient/ITRestClientTest.java | 172 -
.../org/apache/kylin/source/SourceManagerTest.java | 61 -
.../source/hive/ITHiveSourceTableLoaderTest.java | 60 -
.../kylin/source/hive/ITHiveTableReaderTest.java | 49 -
.../kylin/source/hive/ITSnapshotManagerTest.java | 83 -
.../source/jdbc/ITJdbcSourceTableLoaderTest.java | 119 -
.../kylin/source/jdbc/ITJdbcTableReaderTest.java | 114 -
.../storage/hbase/ITAclTableMigrationToolTest.java | 211 -
.../storage/hbase/ITHBaseResourceStoreTest.java | 96 -
.../apache/kylin/storage/hbase/ITHdfsOpsTest.java | 64 -
.../apache/kylin/storage/hbase/ITStorageTest.java | 165 -
.../hbase/ITZookeeperDistributedLockTest.java | 258 -
.../storage/hdfs/ITHDFSResourceStoreTest.java | 58 -
.../storage/jdbc/ITJDBCResourceStoreTest.java | 319 -
kylin-it/src/test/resources/logging.properties | 23 -
kylin-it/src/test/resources/query/sql/query00.sql | 23 -
kylin-it/src/test/resources/query/sql/query01.sql | 21 -
kylin-it/src/test/resources/query/sql/query02.sql | 20 -
kylin-it/src/test/resources/query/sql/query03.sql | 21 -
kylin-it/src/test/resources/query/sql/query04.sql | 22 -
kylin-it/src/test/resources/query/sql/query05.sql | 23 -
kylin-it/src/test/resources/query/sql/query06.sql | 24 -
kylin-it/src/test/resources/query/sql/query07.sql | 23 -
kylin-it/src/test/resources/query/sql/query08.sql | 23 -
kylin-it/src/test/resources/query/sql/query09.sql | 28 -
kylin-it/src/test/resources/query/sql/query10.sql | 31 -
kylin-it/src/test/resources/query/sql/query100.sql | 21 -
kylin-it/src/test/resources/query/sql/query101.sql | 32 -
kylin-it/src/test/resources/query/sql/query102.sql | 32 -
kylin-it/src/test/resources/query/sql/query103.sql | 32 -
kylin-it/src/test/resources/query/sql/query104.sql | 32 -
kylin-it/src/test/resources/query/sql/query105.sql | 26 -
kylin-it/src/test/resources/query/sql/query106.sql | 102 -
kylin-it/src/test/resources/query/sql/query107.sql | 103 -
kylin-it/src/test/resources/query/sql/query108.sql | 43 -
kylin-it/src/test/resources/query/sql/query109.sql | 26 -
kylin-it/src/test/resources/query/sql/query11.sql | 31 -
kylin-it/src/test/resources/query/sql/query110.sql | 59 -
kylin-it/src/test/resources/query/sql/query112.sql | 29 -
kylin-it/src/test/resources/query/sql/query113.sql | 29 -
kylin-it/src/test/resources/query/sql/query114.sql | 29 -
kylin-it/src/test/resources/query/sql/query115.sql | 23 -
kylin-it/src/test/resources/query/sql/query12.sql | 30 -
kylin-it/src/test/resources/query/sql/query13.sql | 29 -
kylin-it/src/test/resources/query/sql/query14.sql | 37 -
kylin-it/src/test/resources/query/sql/query15.sql | 32 -
kylin-it/src/test/resources/query/sql/query16.sql | 36 -
kylin-it/src/test/resources/query/sql/query17.sql | 38 -
kylin-it/src/test/resources/query/sql/query18.sql | 34 -
kylin-it/src/test/resources/query/sql/query19.sql | 38 -
kylin-it/src/test/resources/query/sql/query20.sql | 34 -
kylin-it/src/test/resources/query/sql/query21.sql | 41 -
kylin-it/src/test/resources/query/sql/query22.sql | 43 -
kylin-it/src/test/resources/query/sql/query23.sql | 41 -
kylin-it/src/test/resources/query/sql/query24.sql | 45 -
kylin-it/src/test/resources/query/sql/query25.sql | 46 -
kylin-it/src/test/resources/query/sql/query26.sql | 48 -
kylin-it/src/test/resources/query/sql/query27.sql | 48 -
kylin-it/src/test/resources/query/sql/query28.sql | 31 -
kylin-it/src/test/resources/query/sql/query29.sql | 33 -
kylin-it/src/test/resources/query/sql/query30.sql | 20 -
kylin-it/src/test/resources/query/sql/query31.sql | 41 -
kylin-it/src/test/resources/query/sql/query32.sql | 40 -
kylin-it/src/test/resources/query/sql/query33.sql | 44 -
kylin-it/src/test/resources/query/sql/query34.sql | 42 -
kylin-it/src/test/resources/query/sql/query35.sql | 42 -
kylin-it/src/test/resources/query/sql/query36.sql | 35 -
kylin-it/src/test/resources/query/sql/query37.sql | 37 -
kylin-it/src/test/resources/query/sql/query38.sql | 23 -
kylin-it/src/test/resources/query/sql/query39.sql | 21 -
kylin-it/src/test/resources/query/sql/query40.sql | 23 -
kylin-it/src/test/resources/query/sql/query41.sql | 31 -
kylin-it/src/test/resources/query/sql/query42.sql | 28 -
kylin-it/src/test/resources/query/sql/query43.sql | 30 -
kylin-it/src/test/resources/query/sql/query44.sql | 32 -
kylin-it/src/test/resources/query/sql/query49.sql | 36 -
kylin-it/src/test/resources/query/sql/query50.sql | 30 -
kylin-it/src/test/resources/query/sql/query51.sql | 30 -
kylin-it/src/test/resources/query/sql/query52.sql | 23 -
kylin-it/src/test/resources/query/sql/query53.sql | 25 -
kylin-it/src/test/resources/query/sql/query54.sql | 30 -
kylin-it/src/test/resources/query/sql/query55.sql | 31 -
kylin-it/src/test/resources/query/sql/query56.sql | 31 -
kylin-it/src/test/resources/query/sql/query57.sql | 31 -
kylin-it/src/test/resources/query/sql/query58.sql | 31 -
kylin-it/src/test/resources/query/sql/query59.sql | 20 -
kylin-it/src/test/resources/query/sql/query60.sql | 30 -
kylin-it/src/test/resources/query/sql/query61.sql | 30 -
kylin-it/src/test/resources/query/sql/query62.sql | 22 -
kylin-it/src/test/resources/query/sql/query63.sql | 21 -
kylin-it/src/test/resources/query/sql/query64.sql | 22 -
kylin-it/src/test/resources/query/sql/query65.sql | 20 -
kylin-it/src/test/resources/query/sql/query66.sql | 25 -
kylin-it/src/test/resources/query/sql/query67.sql | 23 -
kylin-it/src/test/resources/query/sql/query68.sql | 27 -
kylin-it/src/test/resources/query/sql/query69.sql | 26 -
kylin-it/src/test/resources/query/sql/query70.sql | 25 -
kylin-it/src/test/resources/query/sql/query71.sql | 25 -
kylin-it/src/test/resources/query/sql/query72.sql | 29 -
kylin-it/src/test/resources/query/sql/query73.sql | 20 -
kylin-it/src/test/resources/query/sql/query74.sql | 20 -
kylin-it/src/test/resources/query/sql/query75.sql | 23 -
kylin-it/src/test/resources/query/sql/query76.sql | 23 -
kylin-it/src/test/resources/query/sql/query77.sql | 22 -
kylin-it/src/test/resources/query/sql/query78.sql | 23 -
kylin-it/src/test/resources/query/sql/query79.sql | 20 -
kylin-it/src/test/resources/query/sql/query80.sql | 29 -
kylin-it/src/test/resources/query/sql/query83.sql | 34 -
kylin-it/src/test/resources/query/sql/query84.sql | 34 -
kylin-it/src/test/resources/query/sql/query85.sql | 26 -
kylin-it/src/test/resources/query/sql/query86.sql | 25 -
kylin-it/src/test/resources/query/sql/query87.sql | 23 -
kylin-it/src/test/resources/query/sql/query88.sql | 23 -
kylin-it/src/test/resources/query/sql/query89.sql | 23 -
kylin-it/src/test/resources/query/sql/query90.sql | 25 -
kylin-it/src/test/resources/query/sql/query91.sql | 25 -
kylin-it/src/test/resources/query/sql/query92.sql | 31 -
kylin-it/src/test/resources/query/sql/query93.sql | 31 -
kylin-it/src/test/resources/query/sql/query94.sql | 31 -
kylin-it/src/test/resources/query/sql/query95.sql | 31 -
kylin-it/src/test/resources/query/sql/query96.sql | 23 -
kylin-it/src/test/resources/query/sql/query97.sql | 29 -
kylin-it/src/test/resources/query/sql/query98.sql | 22 -
kylin-it/src/test/resources/query/sql/query99.sql | 33 -
.../src/test/resources/query/sql_cache/query01.sql | 30 -
.../src/test/resources/query/sql_cache/query02.sql | 30 -
.../src/test/resources/query/sql_cache/query03.sql | 30 -
.../src/test/resources/query/sql_cache/query04.sql | 30 -
.../src/test/resources/query/sql_cache/query05.sql | 30 -
.../src/test/resources/query/sql_cache/query06.sql | 30 -
.../src/test/resources/query/sql_cache/query07.sql | 30 -
.../src/test/resources/query/sql_cache/query08.sql | 30 -
.../src/test/resources/query/sql_cache/query09.sql | 30 -
.../test/resources/query/sql_casewhen/query00.sql | 28 -
.../test/resources/query/sql_casewhen/query01.sql | 27 -
.../test/resources/query/sql_casewhen/query02.sql | 23 -
.../test/resources/query/sql_casewhen/query03.sql | 26 -
.../test/resources/query/sql_casewhen/query04.sql | 35 -
.../test/resources/query/sql_casewhen/query55.sql | 23 -
.../test/resources/query/sql_casewhen/query56.sql | 23 -
.../test/resources/query/sql_casewhen/query57.sql | 24 -
.../test/resources/query/sql_casewhen/query58.sql | 23 -
.../resources/query/sql_castprunesegs/query01.sql | 22 -
.../resources/query/sql_castprunesegs/query02.sql | 22 -
.../resources/query/sql_castprunesegs/query03.sql | 31 -
.../test/resources/query/sql_datetime/query00.sql | 15 -
.../test/resources/query/sql_derived/query01.sql | 48 -
.../test/resources/query/sql_derived/query02.sql | 48 -
.../test/resources/query/sql_derived/query03.sql | 20 -
.../test/resources/query/sql_derived/query04.sql | 20 -
.../test/resources/query/sql_derived/query05.sql | 29 -
.../test/resources/query/sql_derived/query06.sql | 20 -
.../test/resources/query/sql_derived/query07.sql | 23 -
.../test/resources/query/sql_derived/query08.sql | 20 -
.../test/resources/query/sql_derived/query09.sql | 20 -
.../test/resources/query/sql_derived/query10.sql | 45 -
.../test/resources/query/sql_derived/query11.sql | 27 -
.../test/resources/query/sql_derived/query12.sql | 25 -
.../test/resources/query/sql_derived/query13.sql | 27 -
.../query/sql_dict_enumerator/query01.sql | 22 -
.../query/sql_dict_enumerator/query02.sql | 21 -
.../query/sql_dict_enumerator/query03.sql | 21 -
.../test/resources/query/sql_distinct/query00.sql | 25 -
.../query00.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...4638f6-3788-46e2-b662-1868271164b4-c000.csv.crc | Bin 872 -> 0 bytes
.../sql_distinct/query00.sql.expected/_SUCCESS | 0
...0-a34638f6-3788-46e2-b662-1868271164b4-c000.csv | 3427 -
.../test/resources/query/sql_distinct/query01.sql | 26 -
.../query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...219da1-c7c9-4934-8c6f-82b434e27fac-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query01.sql.expected/_SUCCESS | 0
...0-21219da1-c7c9-4934-8c6f-82b434e27fac-c000.csv | 1 -
.../test/resources/query/sql_distinct/query02.sql | 27 -
.../query02.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...399230-d604-4ccf-8cf6-899a065f39a2-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query02.sql.expected/_SUCCESS | 0
...0-85399230-d604-4ccf-8cf6-899a065f39a2-c000.csv | 1 -
.../test/resources/query/sql_distinct/query03.sql | 27 -
.../query03.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...fb3791-2e08-4e43-af9a-2c64d78a3f22-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query03.sql.expected/_SUCCESS | 0
...0-49fb3791-2e08-4e43-af9a-2c64d78a3f22-c000.csv | 13 -
.../test/resources/query/sql_distinct/query04.sql | 28 -
.../query04.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...c5f310-37b7-44d3-a8ce-461be48775d2-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query04.sql.expected/_SUCCESS | 0
...0-59c5f310-37b7-44d3-a8ce-461be48775d2-c000.csv | 13 -
.../test/resources/query/sql_distinct/query05.sql | 26 -
.../query05.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...7b0e0c-49f7-427d-826d-f2a8bc2c62b6-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query05.sql.expected/_SUCCESS | 0
...0-697b0e0c-49f7-427d-826d-f2a8bc2c62b6-c000.csv | 5 -
.../test/resources/query/sql_distinct/query06.sql | 27 -
.../query06.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...8c49ba-f80b-4526-bded-1542272f5248-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query06.sql.expected/_SUCCESS | 0
...0-e98c49ba-f80b-4526-bded-1542272f5248-c000.csv | 1 -
.../test/resources/query/sql_distinct/query07.sql | 25 -
.../query07.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...1b6cf6-4979-40ea-b24b-6e9bba99fc77-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_distinct/query07.sql.expected/_SUCCESS | 0
...0-6c1b6cf6-4979-40ea-b24b-6e9bba99fc77-c000.csv | 5 -
.../resources/query/sql_distinct_dim/query100.sql | 32 -
.../resources/query/sql_distinct_dim/query101.sql | 22 -
.../query/sql_distinct_precisely/query00.sql | 25 -
.../query/sql_distinct_precisely/query01.sql | 26 -
.../query/sql_distinct_precisely/query02.sql | 27 -
.../query/sql_distinct_precisely/query03.sql | 36 -
.../query/sql_distinct_precisely/query04.sql | 37 -
.../sql_distinct_precisely_rollup/query03.sql | 36 -
.../sql_distinct_precisely_rollup/query04.sql | 37 -
.../test/resources/query/sql_dynamic/query01.dat | 2 -
.../test/resources/query/sql_dynamic/query01.sql | 29 -
.../test/resources/query/sql_dynamic/query02.dat | 2 -
.../test/resources/query/sql_dynamic/query02.sql | 31 -
.../test/resources/query/sql_dynamic/query03.dat | 2 -
.../test/resources/query/sql_dynamic/query03.sql | 34 -
.../test/resources/query/sql_dynamic/query04.dat | 2 -
.../test/resources/query/sql_dynamic/query04.sql | 21 -
.../resources/query/sql_exactly_agg/query00.sql | 25 -
.../query00.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...b834cd-5c9e-4c0b-8074-7238869f186c-c000.csv.crc | Bin 124 -> 0 bytes
.../sql_exactly_agg/query00.sql.expected/_SUCCESS | 0
...0-ddb834cd-5c9e-4c0b-8074-7238869f186c-c000.csv | 300 -
.../resources/query/sql_exactly_agg/query01.sql | 25 -
.../query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...3fdd67-7c83-4382-82dd-bb19338fb3f5-c000.csv.crc | Bin 24 -> 0 bytes
.../sql_exactly_agg/query01.sql.expected/_SUCCESS | 0
...0-c43fdd67-7c83-4382-82dd-bb19338fb3f5-c000.csv | 40 -
.../resources/query/sql_exactly_agg/query02.sql | 32 -
.../query02.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...19cdab-4abc-4173-92d1-8d190775b8bd-c000.csv.crc | Bin 240 -> 0 bytes
.../sql_exactly_agg/query02.sql.expected/_SUCCESS | 0
...0-aa19cdab-4abc-4173-92d1-8d190775b8bd-c000.csv | 600 -
.../resources/query/sql_exactly_agg/query03.sql | 43 -
.../query03.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...fae054-7f80-461f-ab4a-bc19d5f6bda3-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_exactly_agg/query03.sql.expected/_SUCCESS | 0
...0-5afae054-7f80-461f-ab4a-bc19d5f6bda3-c000.csv | 5 -
.../resources/query/sql_exactly_agg/query04.sql | 34 -
.../query04.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...12f067-288d-4fa8-8ded-2613ec384f20-c000.csv.crc | Bin 3188 -> 0 bytes
.../sql_exactly_agg/query04.sql.expected/_SUCCESS | 0
...0-7f12f067-288d-4fa8-8ded-2613ec384f20-c000.csv | 6462 -
.../resources/query/sql_exactly_agg/query05.sql | 20 -
.../query05.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...e15acc-11b7-42f9-8faf-913c2d4cbf62-c000.csv.crc | Bin 136 -> 0 bytes
.../sql_exactly_agg/query05.sql.expected/_SUCCESS | 0
...0-17e15acc-11b7-42f9-8faf-913c2d4cbf62-c000.csv | 731 -
.../resources/query/sql_exactly_agg/query06.sql | 25 -
.../query06.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...20bf7e-f8b3-475f-b50b-09916d61637a-c000.csv.crc | Bin 108 -> 0 bytes
.../sql_exactly_agg/query06.sql.expected/_SUCCESS | 0
...0-0220bf7e-f8b3-475f-b50b-09916d61637a-c000.csv | 466 -
.../resources/query/sql_exactly_agg/query07.sql | 25 -
.../query07.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...e1e80c-4fef-4263-9331-273675453c90-c000.csv.crc | Bin 84 -> 0 bytes
.../sql_exactly_agg/query07.sql.expected/_SUCCESS | 0
...0-29e1e80c-4fef-4263-9331-273675453c90-c000.csv | 346 -
.../resources/query/sql_exactly_agg/query08.sql | 25 -
.../query08.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...45836b-bc11-4eac-807d-02c54e8b1337-c000.csv.crc | Bin 92 -> 0 bytes
.../sql_exactly_agg/query08.sql.expected/_SUCCESS | 0
...0-0745836b-bc11-4eac-807d-02c54e8b1337-c000.csv | 404 -
.../resources/query/sql_exactly_agg/query09.sql | 28 -
.../query09.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...31db76-924c-4045-bd66-369c632ce1e1-c000.csv.crc | Bin 124 -> 0 bytes
.../sql_exactly_agg/query09.sql.expected/_SUCCESS | 0
...0-ca31db76-924c-4045-bd66-369c632ce1e1-c000.csv | 300 -
.../resources/query/sql_exactly_agg/query10.sql | 21 -
.../query10.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...c05bff-a47f-46e0-afcc-7bb0652e6880-c000.csv.crc | Bin 12 -> 0 bytes
.../sql_exactly_agg/query10.sql.expected/_SUCCESS | 0
...0-87c05bff-a47f-46e0-afcc-7bb0652e6880-c000.csv | 1 -
.../resources/query/sql_exactly_agg/query11.sql | 21 -
.../sql_exactly_agg_multi_segment/query00.sql | 24 -
.../sql_exactly_agg_multi_segment/query01.sql | 24 -
.../resources/query/sql_expression/query01.sql | 22 -
.../resources/query/sql_expression/query02.sql | 34 -
.../resources/query/sql_expression/query03.sql | 26 -
.../resources/query/sql_expression/query04.sql | 40 -
.../resources/query/sql_expression/query05.sql | 40 -
.../resources/query/sql_expression/query06.sql | 43 -
.../test/resources/query/sql_function/query01.sql | 24 -
.../test/resources/query/sql_grouping/query00.sql | 26 -
.../test/resources/query/sql_grouping/query01.sql | 26 -
.../test/resources/query/sql_grouping/query02.sql | 26 -
.../test/resources/query/sql_grouping/query03.sql | 26 -
.../test/resources/query/sql_grouping/query04.sql | 26 -
.../test/resources/query/sql_grouping/query05.sql | 26 -
.../test/resources/query/sql_grouping/query06.sql | 26 -
.../test/resources/query/sql_grouping/query07.sql | 24 -
.../test/resources/query/sql_grouping/query08.sql | 24 -
.../test/resources/query/sql_grouping/query09.sql | 24 -
.../src/test/resources/query/sql_h2/query03.sql | 23 -
.../src/test/resources/query/sql_h2/query04.sql | 24 -
.../src/test/resources/query/sql_h2/query05.sql | 25 -
.../sql_h2/query05.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...e62711-bb46-4c98-94ad-17ed4ae32386-c000.csv.crc | Bin 164 -> 0 bytes
.../query/sql_h2/query05.sql.expected/_SUCCESS | 0
...0-6ae62711-bb46-4c98-94ad-17ed4ae32386-c000.csv | 731 -
.../src/test/resources/query/sql_h2/query07.sql | 22 -
.../src/test/resources/query/sql_h2/query09.sql | 24 -
.../src/test/resources/query/sql_h2/query10.sql | 26 -
.../src/test/resources/query/sql_hive/query01.sql | 21 -
.../src/test/resources/query/sql_hive/query10.sql | 27 -
.../query/sql_intersect_count/query00.sql | 32 -
.../query00.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...e31c51-c35a-4d88-9eea-e3ef825967d8-c000.csv.crc | Bin 12 -> 0 bytes
.../query00.sql.expected/_SUCCESS | 0
...0-24e31c51-c35a-4d88-9eea-e3ef825967d8-c000.csv | 2 -
.../query/sql_intersect_count/query01.sql | 27 -
.../query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...c77c16-39dd-489d-9078-2a12bb25ef3f-c000.csv.crc | Bin 12 -> 0 bytes
.../query01.sql.expected/_SUCCESS | 0
...0-09c77c16-39dd-489d-9078-2a12bb25ef3f-c000.csv | 3 -
.../query/sql_intersect_count/query03.sql | 23 -
.../query03.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...535c4a-649e-4a27-8ee3-f839e858e96b-c000.csv.crc | Bin 12 -> 0 bytes
.../query03.sql.expected/_SUCCESS | 0
...0-0c535c4a-649e-4a27-8ee3-f839e858e96b-c000.csv | 28 -
.../query/sql_intersect_count/query04.sql | 33 -
.../query04.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...a75ff1-920d-4161-a3ad-080d3e71ce25-c000.csv.crc | Bin 12 -> 0 bytes
.../query04.sql.expected/_SUCCESS | 0
...0-0ba75ff1-920d-4161-a3ad-080d3e71ce25-c000.csv | 2 -
.../query/sql_intersect_count/query05.sql | 23 -
.../query05.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...ed9048-d2e2-414a-8588-9ed588204a97-c000.csv.crc | Bin 12 -> 0 bytes
.../query05.sql.expected/_SUCCESS | 0
...0-a3ed9048-d2e2-414a-8588-9ed588204a97-c000.csv | 1 -
.../query/sql_intersect_value/query00.sql | 32 -
.../query00.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...6bc9a2-0d5a-4839-94e1-d2ec55db7c1b-c000.csv.crc | Bin 12 -> 0 bytes
.../query00.sql.expected/_SUCCESS | 0
...0-3a6bc9a2-0d5a-4839-94e1-d2ec55db7c1b-c000.csv | 2 -
.../query/sql_intersect_value/query01.sql | 27 -
.../query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...3c8db3-f109-43f8-b8bc-c02ff5f0405c-c000.csv.crc | Bin 12 -> 0 bytes
.../query01.sql.expected/_SUCCESS | 0
...0-513c8db3-f109-43f8-b8bc-c02ff5f0405c-c000.csv | 3 -
.../query/sql_intersect_value/query03.sql | 23 -
.../query03.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...c6dcd8-9969-4209-bd1e-b5906fe175c6-c000.csv.crc | Bin 12 -> 0 bytes
.../query03.sql.expected/_SUCCESS | 0
...0-77c6dcd8-9969-4209-bd1e-b5906fe175c6-c000.csv | 28 -
.../query/sql_intersect_value/query04.sql | 33 -
.../query04.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...2bf65a-af47-44ab-84b2-56abc974a2b0-c000.csv.crc | Bin 12 -> 0 bytes
.../query04.sql.expected/_SUCCESS | 0
...0-892bf65a-af47-44ab-84b2-56abc974a2b0-c000.csv | 2 -
.../query/sql_intersect_value/query05.sql | 23 -
.../query05.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...3de84f-1d22-4777-8876-908c30a97290-c000.csv.crc | Bin 12 -> 0 bytes
.../query05.sql.expected/_SUCCESS | 0
...0-bf3de84f-1d22-4777-8876-908c30a97290-c000.csv | 1 -
.../query/sql_intersect_value/query06.sql | 29 -
.../query06.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...e47126-09a8-4b4c-b022-a3c537ab5c04-c000.csv.crc | Bin 12 -> 0 bytes
.../query06.sql.expected/_SUCCESS | 0
...0-49e47126-09a8-4b4c-b022-a3c537ab5c04-c000.csv | 2 -
.../src/test/resources/query/sql_join/query_00.sql | 22 -
.../src/test/resources/query/sql_like/query01.sql | 32 -
.../src/test/resources/query/sql_like/query02.sql | 23 -
.../src/test/resources/query/sql_like/query03.sql | 34 -
.../src/test/resources/query/sql_like/query04.sql | 23 -
.../src/test/resources/query/sql_like/query05.sql | 34 -
.../src/test/resources/query/sql_like/query06.sql | 34 -
.../src/test/resources/query/sql_like/query10.sql | 13 -
.../src/test/resources/query/sql_like/query15.sql | 13 -
.../src/test/resources/query/sql_like/query16.sql | 13 -
.../src/test/resources/query/sql_like/query17.sql | 13 -
.../src/test/resources/query/sql_like/query18.sql | 13 -
.../src/test/resources/query/sql_like/query20.sql | 32 -
.../src/test/resources/query/sql_like/query21.sql | 32 -
.../src/test/resources/query/sql_like/query22.sql | 32 -
.../src/test/resources/query/sql_like/query23.sql | 32 -
.../src/test/resources/query/sql_limit/query00.sql | 24 -
.../sql_limit/query00.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...90bf53-2025-46e5-8133-2a6bf503206c-c000.csv.crc | Bin 12 -> 0 bytes
.../query/sql_limit/query00.sql.expected/_SUCCESS | 0
...0-a290bf53-2025-46e5-8133-2a6bf503206c-c000.csv | 6 -
.../test/resources/query/sql_lookup/query01.sql | 20 -
.../test/resources/query/sql_lookup/query02.sql | 20 -
.../test/resources/query/sql_lookup/query03.sql | 20 -
.../test/resources/query/sql_lookup/query04.sql | 20 -
.../test/resources/query/sql_lookup/query05.sql | 20 -
.../test/resources/query/sql_lookup/query06.sql | 20 -
.../test/resources/query/sql_lookup/query07.sql | 20 -
.../test/resources/query/sql_lookup/query45.sql | 20 -
.../test/resources/query/sql_lookup/query46.sql | 20 -
.../test/resources/query/sql_lookup/query47.sql | 20 -
.../test/resources/query/sql_lookup/query48.sql | 20 -
.../test/resources/query/sql_lookup/query55.sql | 20 -
.../resources/query/sql_multi_model/query01.sql | 50 -
.../query/sql_non_equal_join/query_00.sql | 11 -
.../query/sql_non_equal_join/query_01.sql | 12 -
.../query/sql_non_equal_join/query_02.sql | 17 -
.../test/resources/query/sql_orderby/query01.sql | 33 -
.../test/resources/query/sql_orderby/query02.sql | 26 -
.../test/resources/query/sql_orderby/query03.sql | 25 -
.../test/resources/query/sql_ordinal/query01.sql | 33 -
.../test/resources/query/sql_ordinal/query02.sql | 25 -
.../resources/query/sql_percentile/query01.sql | 21 -
.../query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...74fce0-2455-4b7d-baa1-91c4bed92238-c000.csv.crc | Bin 220 -> 0 bytes
.../sql_percentile/query01.sql.expected/_SUCCESS | 0
...0-5f74fce0-2455-4b7d-baa1-91c4bed92238-c000.csv | 1000 -
.../resources/query/sql_percentile/query02.sql | 21 -
.../query02.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...5b7736-3444-4bbb-b4e1-accdbc25c404-c000.csv.crc | Bin 220 -> 0 bytes
.../sql_percentile/query02.sql.expected/_SUCCESS | 0
...0-435b7736-3444-4bbb-b4e1-accdbc25c404-c000.csv | 1000 -
.../resources/query/sql_percentile/query03.sql | 20 -
.../src/test/resources/query/sql_plan/query01.sql | 27 -
.../sql_plan/query01.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...d6687e-124c-4090-8cc6-36c9aa61c660-c000.csv.crc | Bin 276 -> 0 bytes
.../query/sql_plan/query01.sql.expected/_SUCCESS | 0
...0-6cd6687e-124c-4090-8cc6-36c9aa61c660-c000.csv | 7173 -
.../src/test/resources/query/sql_plan/query02.sql | 24 -
.../sql_plan/query02.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...d3b74d-27b2-4437-a05a-4b066f84b43e-c000.csv.crc | Bin 276 -> 0 bytes
.../query/sql_plan/query02.sql.expected/_SUCCESS | 0
...0-a3d3b74d-27b2-4437-a05a-4b066f84b43e-c000.csv | 7173 -
.../resources/query/sql_prune_segment/query00.sql | 20 -
.../resources/query/sql_prune_segment/query01.sql | 20 -
.../resources/query/sql_prune_segment/query02.sql | 28 -
.../query02.sql.expected/._SUCCESS.crc | Bin 8 -> 0 bytes
...343019-e3fd-4ce5-b509-d744f9ccb327-c000.csv.crc | Bin 12 -> 0 bytes
.../query02.sql.expected/_SUCCESS | 0
...0-45343019-e3fd-4ce5-b509-d744f9ccb327-c000.csv | 3 -
.../test/resources/query/sql_snowflake/query01.sql | 42 -
.../test/resources/query/sql_snowflake/query02.sql | 32 -
.../test/resources/query/sql_snowflake/query03.sql | 33 -
.../test/resources/query/sql_snowflake/query04.sql | 38 -
.../test/resources/query/sql_snowflake/query05.sql | 39 -
.../test/resources/query/sql_snowflake/query06.sql | 40 -
.../test/resources/query/sql_streaming/query01.sql | 19 -
.../test/resources/query/sql_streaming/query02.sql | 19 -
.../test/resources/query/sql_streaming/query03.sql | 19 -
.../test/resources/query/sql_streaming/query04.sql | 19 -
.../test/resources/query/sql_streaming/query05.sql | 19 -
.../test/resources/query/sql_streaming/query06.sql | 19 -
.../test/resources/query/sql_streaming/query07.sql | 19 -
.../test/resources/query/sql_streaming/query08.sql | 19 -
.../test/resources/query/sql_streaming/query09.sql | 19 -
.../test/resources/query/sql_streaming/query10.sql | 19 -
.../test/resources/query/sql_streaming/query11.sql | 19 -
.../test/resources/query/sql_streaming/query12.sql | 19 -
.../test/resources/query/sql_streaming/query13.sql | 19 -
.../test/resources/query/sql_streaming/query14.sql | 19 -
.../sql_streaming_v2/compare_result/query01.sql | 19 -
.../sql_streaming_v2/compare_result/query02.sql | 19 -
.../sql_streaming_v2/compare_result/query03.sql | 19 -
.../sql_streaming_v2/compare_result/query04.sql | 19 -
.../sql_streaming_v2/compare_result/query05.sql | 19 -
.../sql_streaming_v2/compare_result/query06.sql | 19 -
.../sql_streaming_v2/compare_result/query07.sql | 19 -
.../sql_streaming_v2/compare_result/query08.sql | 19 -
.../query/sql_streaming_v2/count/query01.sql | 19 -
.../not_compare_result/query01.sql | 20 -
.../not_compare_result/query02.sql | 20 -
.../test/resources/query/sql_subquery/query00.sql | 28 -
.../test/resources/query/sql_subquery/query01.sql | 32 -
.../test/resources/query/sql_subquery/query02.sql | 36 -
.../test/resources/query/sql_subquery/query03.sql | 37 -
.../test/resources/query/sql_subquery/query04.sql | 37 -
.../test/resources/query/sql_subquery/query05.sql | 38 -
.../test/resources/query/sql_subquery/query06.sql | 25 -
.../test/resources/query/sql_subquery/query07.sql | 25 -
.../test/resources/query/sql_subquery/query08.sql | 43 -
.../test/resources/query/sql_subquery/query09.sql | 44 -
.../test/resources/query/sql_subquery/query10.sql | 29 -
.../test/resources/query/sql_subquery/query11.sql | 26 -
.../test/resources/query/sql_subquery/query12.sql | 38 -
.../test/resources/query/sql_subquery/query13.sql | 26 -
.../test/resources/query/sql_subquery/query14.sql | 41 -
.../test/resources/query/sql_subquery/query15.sql | 30 -
.../test/resources/query/sql_subquery/query16.sql | 39 -
.../test/resources/query/sql_subquery/query17.sql | 36 -
.../test/resources/query/sql_subquery/query18.sql | 40 -
.../test/resources/query/sql_subquery/query19.sql | 37 -
.../test/resources/query/sql_subquery/query21.sql | 43 -
.../test/resources/query/sql_subquery/query22.sql | 42 -
.../test/resources/query/sql_subquery/query23.sql | 43 -
.../test/resources/query/sql_subquery/query24.sql | 46 -
.../test/resources/query/sql_subquery/query25.sql | 39 -
.../test/resources/query/sql_subquery/query26.sql | 35 -
.../test/resources/query/sql_subquery/query27.sql | 37 -
.../test/resources/query/sql_subquery/query28.sql | 38 -
.../test/resources/query/sql_subquery/query29.sql | 39 -
.../test/resources/query/sql_subquery/query30.sql | 33 -
.../test/resources/query/sql_subquery/query31.sql | 37 -
.../test/resources/query/sql_subquery/query32.sql | 32 -
.../test/resources/query/sql_subquery/query33.sql | 32 -
.../test/resources/query/sql_subquery/query34.sql | 32 -
.../test/resources/query/sql_subquery/query35.sql | 26 -
.../test/resources/query/sql_subquery/query36.sql | 24 -
.../test/resources/query/sql_subquery/query37.sql | 32 -
.../test/resources/query/sql_tableau/query00.sql | 25 -
.../test/resources/query/sql_tableau/query01.sql | 33 -
.../test/resources/query/sql_tableau/query02.sql | 31 -
.../test/resources/query/sql_tableau/query03.sql | 36 -
.../test/resources/query/sql_tableau/query04.sql | 40 -
.../test/resources/query/sql_tableau/query05.sql | 41 -
.../test/resources/query/sql_tableau/query06.sql | 41 -
.../test/resources/query/sql_tableau/query07.sql | 27 -
.../test/resources/query/sql_tableau/query08.sql | 25 -
.../test/resources/query/sql_tableau/query09.sql | 30 -
.../test/resources/query/sql_tableau/query10.sql | 26 -
.../test/resources/query/sql_tableau/query11.sql | 26 -
.../test/resources/query/sql_tableau/query12.sql | 26 -
.../test/resources/query/sql_tableau/query13.sql | 25 -
.../test/resources/query/sql_tableau/query14.sql | 25 -
.../test/resources/query/sql_tableau/query15.sql | 25 -
.../test/resources/query/sql_tableau/query16.sql | 25 -
.../test/resources/query/sql_tableau/query17.sql | 25 -
.../test/resources/query/sql_tableau/query18.sql | 22 -
.../test/resources/query/sql_tableau/query19.sql | 27 -
.../test/resources/query/sql_tableau/query20.sql | 27 -
.../test/resources/query/sql_tableau/query21.sql | 28 -
.../test/resources/query/sql_tableau/query22.sql | 28 -
.../test/resources/query/sql_tableau/query23.sql | 24 -
.../test/resources/query/sql_tableau/query24.sql | 25 -
.../test/resources/query/sql_tableau/query25.sql | 29 -
.../test/resources/query/sql_tableau/query27.sql | 25 -
.../test/resources/query/sql_tableau/query28.sql | 24 -
.../test/resources/query/sql_tableau/query29.sql | 30 -
.../test/resources/query/sql_tableau/query67.sql | 24 -
.../test/resources/query/sql_timeout/query02.sql | 21 -
.../test/resources/query/sql_timestamp/query01.sql | 30 -
.../test/resources/query/sql_timestamp/query02.sql | 30 -
.../test/resources/query/sql_timestamp/query03.sql | 30 -
.../test/resources/query/sql_timestamp/query04.sql | 32 -
.../test/resources/query/sql_timestamp/query05.sql | 30 -
.../test/resources/query/sql_timestamp/query11.sql | 23 -
.../test/resources/query/sql_timestamp/query12.sql | 23 -
.../test/resources/query/sql_timestamp/query13.sql | 23 -
.../test/resources/query/sql_timestamp/query14.sql | 23 -
.../test/resources/query/sql_timestamp/query21.sql | 30 -
.../resources/query/sql_timestamp/query21_a.sql | 30 -
.../test/resources/query/sql_timestamp/query22.sql | 30 -
.../resources/query/sql_timestamp/query22_a.sql | 30 -
.../resources/query/sql_timestamp/query22_b.sql | 30 -
.../resources/query/sql_timestamp/query22_c.sql | 30 -
.../resources/query/sql_timestamp/query22_d.sql | 30 -
.../resources/query/sql_timestamp/query22_e.sql | 30 -
.../test/resources/query/sql_timestamp/query23.sql | 30 -
.../resources/query/sql_timestamp/query23_a.sql | 30 -
.../test/resources/query/sql_timestamp/query24.sql | 32 -
.../resources/query/sql_timestamp/query24_a.sql | 31 -
.../test/resources/query/sql_timestamp/query25.sql | 30 -
.../resources/query/sql_timestamp/query25_a.sql | 30 -
.../test/resources/query/sql_timestamp/query26.sql | 26 -
.../src/test/resources/query/sql_topn/query45.sql | 24 -
.../src/test/resources/query/sql_topn/query81.sql | 29 -
.../src/test/resources/query/sql_topn/query82.sql | 28 -
.../src/test/resources/query/sql_topn/query83.sql | 30 -
.../src/test/resources/query/sql_union/query01.sql | 32 -
.../src/test/resources/query/sql_union/query02.sql | 29 -
.../src/test/resources/query/sql_union/query03.sql | 43 -
.../test/resources/query/sql_unionall/query01.sql | 26 -
.../test/resources/query/sql_unionall/query02.sql | 27 -
.../test/resources/query/sql_unionall/query03.sql | 29 -
.../test/resources/query/sql_values/query01.sql | 53 -
.../test/resources/query/sql_values/query02.sql | 28 -
.../test/resources/query/sql_values/query03.sql | 36 -
.../test/resources/query/sql_values/query04.sql | 69 -
.../test/resources/query/sql_values/query05.sql | 22 -
.../test/resources/query/sql_window/query00.sql | 21 -
.../test/resources/query/sql_window/query01.sql | 21 -
.../test/resources/query/sql_window/query02.sql | 21 -
.../test/resources/query/sql_window/query03.sql | 24 -
.../test/resources/query/sql_window/query04.sql | 23 -
.../test/resources/query/sql_window/query05.sql | 27 -
.../test/resources/query/sql_window/query06.sql | 24 -
.../test/resources/query/sql_window/query07.sql | 24 -
.../test/resources/query/sql_window/query08.sql | 24 -
.../test/resources/query/sql_window/query09.sql | 27 -
.../test/resources/query/sql_window/query11.sql | 24 -
.../test/resources/query/sql_window/query12.sql | 26 -
.../test/resources/query/sql_window/query13.sql | 23 -
kylin-it/src/test/resources/statistics.txt | 4092 -
.../resources/streaming_v2_user_info_messages.txt | 10000 -
.../src/test/scala/kafka/admin/AdminUtils.scala | 730 -
pom.xml | 109 -
source-jdbc/DEPRECATED_MODULE | 19 -
source-jdbc/pom.xml | 121 -
.../org/apache/kylin/source/jdbc/HiveCmdStep.java | 77 -
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 315 -
.../kylin/source/jdbc/JdbcHiveInputBase.java | 519 -
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 61 -
.../kylin/source/jdbc/JdbcHiveSparkInput.java | 48 -
.../org/apache/kylin/source/jdbc/JdbcSource.java | 83 -
.../org/apache/kylin/source/jdbc/JdbcTable.java | 65 -
.../apache/kylin/source/jdbc/JdbcTableReader.java | 124 -
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 166 -
.../kylin/source/jdbc/extensible/JdbcExplorer.java | 201 -
.../source/jdbc/extensible/JdbcHiveInputBase.java | 135 -
.../source/jdbc/extensible/JdbcHiveMRInput.java | 67 -
.../source/jdbc/extensible/JdbcHiveSparkInput.java | 55 -
.../kylin/source/jdbc/extensible/JdbcSource.java | 97 -
.../kylin/source/jdbc/extensible/JdbcTable.java | 68 -
.../source/jdbc/extensible/JdbcTableReader.java | 108 -
.../source/jdbc/metadata/DefaultJdbcMetadata.java | 81 -
.../kylin/source/jdbc/metadata/IJdbcMetadata.java | 37 -
.../source/jdbc/metadata/JdbcMetadataFactory.java | 40 -
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 73 -
.../jdbc/metadata/PostgresqlJdbcMetadata.java | 68 -
.../jdbc/metadata/SQLServerJdbcMetadata.java | 68 -
.../kylin/source/jdbc/sqoop/SqoopCmdStep.java | 100 -
.../apache/kylin/source/jdbc/JdbcExplorerTest.java | 157 -
.../kylin/source/jdbc/JdbcHiveInputBaseTest.java | 68 -
.../org/apache/kylin/source/jdbc/SqlUtilTest.java | 63 -
.../source/jdbc/extensible/JdbcExplorerTest.java | 111 -
.../jdbc/extensible/JdbcHiveMRInputTest.java | 143 -
.../source/jdbc/extensible/JdbcSourceTest.java | 62 -
.../source/jdbc/extensible/JdbcTableTest.java | 58 -
.../kylin/source/jdbc/extensible/TestBase.java | 66 -
.../jdbc/metadata/DefaultJdbcMetadataTest.java | 126 -
.../jdbc/metadata/MySQLJdbcMetadataTest.java | 99 -
.../jdbc/metadata/SQLServerJdbcMetadataTest.java | 68 -
.../ut_meta/jdbc_source/cube/ut_jdbc_shard.json | 10 -
.../jdbc_source/cube_desc/ut_jdbc_shard.json | 558 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
source-kafka/.settings/org.eclipse.jdt.core.prefs | 386 -
source-kafka/.settings/org.eclipse.jdt.ui.prefs | 7 -
source-kafka/DEPRECATED_MODULE | 19 -
source-kafka/pom.xml | 74 -
.../kylin/source/kafka/AbstractTimeParser.java | 36 -
.../apache/kylin/source/kafka/DateTimeParser.java | 62 -
.../kylin/source/kafka/DefaultTimeParser.java | 51 -
.../kylin/source/kafka/KafkaConfigManager.java | 156 -
.../apache/kylin/source/kafka/KafkaInputBase.java | 186 -
.../apache/kylin/source/kafka/KafkaMRInput.java | 119 -
.../org/apache/kylin/source/kafka/KafkaSource.java | 278 -
.../apache/kylin/source/kafka/KafkaSparkInput.java | 53 -
.../apache/kylin/source/kafka/StreamingParser.java | 160 -
.../kylin/source/kafka/TimedJsonStreamParser.java | 203 -
.../kylin/source/kafka/config/BrokerConfig.java | 63 -
.../source/kafka/config/KafkaClusterConfig.java | 55 -
.../kylin/source/kafka/config/KafkaConfig.java | 183 -
.../kafka/config/KafkaConsumerProperties.java | 157 -
.../source/kafka/hadoop/KafkaFlatTableJob.java | 178 -
.../source/kafka/hadoop/KafkaFlatTableMapper.java | 100 -
.../source/kafka/hadoop/KafkaInputFormat.java | 110 -
.../kafka/hadoop/KafkaInputRecordReader.java | 176 -
.../kylin/source/kafka/hadoop/KafkaInputSplit.java | 102 -
.../kylin/source/kafka/job/MergeOffsetStep.java | 83 -
.../kafka/model/StreamCubeFactTableDesc.java | 132 -
.../kylin/source/kafka/util/KafkaClient.java | 143 -
.../source/kafka/util/KafkaSampleProducer.java | 145 -
.../source/kafka/TimedJsonStreamParserTest.java | 176 -
.../kafka/config/KafkaConsumerPropertiesTest.java | 70 -
.../kafka/hadoop/KafkaInputRecordReaderTest.java | 60 -
.../src/test/resources/kylin-kafka-consumer.xml | 33 -
source-kafka/src/test/resources/message.json | 42 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
storage-hbase/.settings/org.eclipse.jdt.core.prefs | 386 -
storage-hbase/.settings/org.eclipse.jdt.ui.prefs | 7 -
storage-hbase/DEPRECATED_MODULE | 19 -
storage-hbase/pom.xml | 177 -
.../kylin/storage/hbase/HBaseConnection.java | 411 -
.../kylin/storage/hbase/HBaseResourceStore.java | 512 -
.../apache/kylin/storage/hbase/HBaseStorage.java | 100 -
.../storage/hbase/common/coprocessor/AggrKey.java | 133 -
.../hbase/common/coprocessor/AggregationCache.java | 89 -
.../common/coprocessor/CoprocessorFilter.java | 80 -
.../common/coprocessor/CoprocessorProjector.java | 116 -
.../common/coprocessor/CoprocessorRowType.java | 146 -
.../hbase/common/coprocessor/FilterDecorator.java | 220 -
.../storage/hbase/cube/HBaseScannerBenchmark.java | 178 -
.../kylin/storage/hbase/cube/SimpleHBaseStore.java | 209 -
.../storage/hbase/cube/v2/CellListIterator.java | 28 -
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 578 -
.../kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 319 -
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 219 -
.../storage/hbase/cube/v2/CubeStorageQuery.java | 45 -
.../hbase/cube/v2/ExpectedSizeIterator.java | 108 -
.../storage/hbase/cube/v2/HBaseReadonlyStore.java | 186 -
.../kylin/storage/hbase/cube/v2/RawScan.java | 123 -
.../storage/hbase/cube/v2/SegmentQueryCache.java | 80 -
.../storage/hbase/cube/v2/SegmentQueryResult.java | 101 -
.../v2/coprocessor/endpoint/CubeVisitService.java | 461 -
.../endpoint/generated/CubeVisitProtos.java | 5767 -
.../coprocessor/endpoint/protobuf/CubeVisit.proto | 78 -
.../storage/hbase/lookup/HBaseLookupMRSteps.java | 180 -
.../hbase/lookup/HBaseLookupMaterializer.java | 40 -
.../storage/hbase/lookup/HBaseLookupProvider.java | 58 -
.../hbase/lookup/HBaseLookupRowEncoder.java | 134 -
.../storage/hbase/lookup/HBaseLookupTable.java | 130 -
.../hbase/lookup/KVSortReducerWithDupKeyCheck.java | 62 -
.../hbase/lookup/LookupTableHFilesBulkLoadJob.java | 106 -
.../hbase/lookup/LookupTableToHFileJob.java | 290 -
.../hbase/lookup/LookupTableToHFileMapper.java | 111 -
.../UpdateSnapshotCacheForQueryServersStep.java | 106 -
.../kylin/storage/hbase/steps/BulkLoadJob.java | 115 -
.../kylin/storage/hbase/steps/CreateHTableJob.java | 346 -
.../kylin/storage/hbase/steps/CubeHFileJob.java | 174 -
.../kylin/storage/hbase/steps/CubeHFileMapper.java | 98 -
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 210 -
.../hbase/steps/HBaseFlinkOutputTransition.java | 97 -
.../kylin/storage/hbase/steps/HBaseJobSteps.java | 284 -
.../hbase/steps/HBaseMROutput2Transition.java | 214 -
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 62 -
.../hbase/steps/HBaseSparkOutputTransition.java | 97 -
.../kylin/storage/hbase/steps/HBaseSparkSteps.java | 85 -
.../hbase/steps/HDFSPathGarbageCollectionStep.java | 140 -
.../storage/hbase/steps/HFileOutputFormat3.java | 674 -
.../kylin/storage/hbase/steps/KeyValueCreator.java | 95 -
.../kylin/storage/hbase/steps/MergeGCStep.java | 131 -
.../kylin/storage/hbase/steps/RowKeyWritable.java | 111 -
.../kylin/storage/hbase/steps/RowValueDecoder.java | 144 -
.../kylin/storage/hbase/steps/SparkCubeHFile.java | 285 -
.../kylin/storage/hbase/util/CleanHtableCLI.java | 77 -
.../kylin/storage/hbase/util/CubeMigrationCLI.java | 604 -
.../storage/hbase/util/CubeMigrationCheckCLI.java | 214 -
.../storage/hbase/util/DeployCoprocessorCLI.java | 576 -
.../hbase/util/DeployCoprocessorCLIOps.java | 49 -
.../storage/hbase/util/ExtendCubeToHybridCLI.java | 260 -
.../hbase/util/GridTableHBaseBenchmark.java | 395 -
.../kylin/storage/hbase/util/HBaseClean.java | 123 -
.../hbase/util/HBaseRegionSizeCalculator.java | 143 -
.../kylin/storage/hbase/util/HBaseUnionUtil.java | 45 -
.../kylin/storage/hbase/util/HBaseUsage.java | 72 -
.../storage/hbase/util/HtableAlterMetadataCLI.java | 95 -
.../storage/hbase/util/OrphanHBaseCleanJob.java | 132 -
.../kylin/storage/hbase/util/PingHBaseCLI.java | 85 -
.../kylin/storage/hbase/util/PrintHBaseConfig.java | 62 -
.../apache/kylin/storage/hbase/util/Results.java | 49 -
.../kylin/storage/hbase/util/RowCounterCLI.java | 90 -
.../kylin/storage/hbase/util/StorageCleanUtil.java | 93 -
.../storage/hbase/util/StorageCleanupJob.java | 365 -
.../storage/hbase/util/UpdateHTableHostCLI.java | 192 -
.../storage/hbase/util/ZookeeperAclBuilder.java | 119 -
.../common/TimeConditionLiteralsReplacerTest.java | 54 -
.../hbase/common/coprocessor/FilterBaseTest.java | 259 -
.../common/coprocessor/FilterEvaluateTest.java | 182 -
.../common/coprocessor/FilterSerializeTest.java | 168 -
.../hbase/common/coprocessor/RowProjectorTest.java | 88 -
.../hbase/cube/MeasureTypeOnlyAggrInBaseTest.java | 121 -
.../storage/hbase/cube/SegmentQueryResultTest.java | 112 -
.../coprocessor/endpoint/CubeVisitServiceTest.java | 569 -
.../hbase/lookup/HBaseLookupRowEncoderTest.java | 98 -
.../hbase/lookup/LookupTableToHFileJobTest.java | 110 -
...UpdateSnapshotCacheForQueryServersStepTest.java | 55 -
.../storage/hbase/steps/CubeHFileMapper2Test.java | 87 -
.../storage/hbase/steps/CubeHFileMapperTest.java | 81 -
.../storage/hbase/steps/HFilePartitionerTest.java | 65 -
.../storage/hbase/steps/MockupMapContext.java | 322 -
.../storage/hbase/steps/RowValueDecoderTest.java | 95 -
.../storage/hbase/steps/SandboxMetastoreCLI.java | 73 -
.../kylin/storage/hbase/steps/TestHbaseClient.java | 101 -
.../hbase/util/DeployCoprocessorCLITest.java | 44 -
.../hbase/util/ZookeeperAclBuilderTest.java | 97 -
.../src/test/resources/data/8d_cuboid/part-r-00000 | Bin 1476517 -> 0 bytes
.../test/resources/data/base_cuboid/part-r-00000 | Bin 394644 -> 0 bytes
.../test/resources/partition_list/_partition.lst | Bin 326 -> 0 bytes
.../src/test/resources/partition_list/part-r-00000 | Bin 15004 -> 0 bytes
storage-stream/DEPRECATED_MODULE | 19 -
storage-stream/pom.xml | 93 -
.../apache/kylin/storage/stream/StreamStorage.java | 72 -
.../kylin/storage/stream/StreamStorageQuery.java | 130 -
.../stream/rpc/HttpStreamDataSearchClient.java | 329 -
.../stream/rpc/IStreamDataSearchClient.java | 45 -
.../stream/rpc/MockedStreamDataSearchClient.java | 40 -
stream-coordinator/DEPRECATED_MODULE | 19 -
stream-coordinator/pom.xml | 110 -
.../kylin/stream/coordinator/Coordinator.java | 1504 -
.../coordinator/HBaseStreamMetadataStore.java | 178 -
.../coordinator/MockStreamMetadataStore.java | 178 -
.../stream/coordinator/StreamMetadataStore.java | 141 -
.../coordinator/StreamMetadataStoreFactory.java | 61 -
.../stream/coordinator/StreamingCubeInfo.java | 66 -
.../kylin/stream/coordinator/StreamingUtils.java | 34 -
.../coordinator/ZookeeperStreamMetadataStore.java | 688 -
.../kylin/stream/coordinator/assign/Assigner.java | 49 -
.../stream/coordinator/assign/AssignmentUtil.java | 87 -
.../coordinator/assign/AssignmentsCache.java | 103 -
.../assign/CubePartitionRoundRobinAssigner.java | 322 -
.../stream/coordinator/assign/DefaultAssigner.java | 212 -
.../coordinator/client/CoordinatorClient.java | 71 -
.../client/CoordinatorClientFactory.java | 66 -
.../coordinator/client/CoordinatorResponse.java | 58 -
.../coordinator/client/HttpCoordinatorClient.java | 298 -
.../coordinator/coordinate/BuildJobSubmitter.java | 533 -
.../coordinate/ReceiverClusterManager.java | 660 -
.../coordinate/SegmentJobBuildInfo.java | 73 -
.../coordinate/StreamingCoordinator.java | 645 -
.../coordinate/annotations/NonSideEffect.java | 41 -
.../annotations/NotAtomicAndNotIdempotent.java | 45 -
.../annotations/NotAtomicIdempotent.java | 42 -
.../stream/coordinator/doctor/ClusterDoctor.java | 33 -
.../coordinator/doctor/ClusterStateChecker.java | 82 -
.../exception/ClusterStateException.java | 63 -
.../coordinator/exception/CoordinateException.java | 42 -
.../exception/NotLeadCoordinatorException.java | 42 -
.../coordinator/exception/StoreException.java | 38 -
.../src/main/resources/log4j.properties | 24 -
.../kylin/stream/coordinator/CoordinatorTest.java | 279 -
.../CubePartitionRoundRobinAssignerTest.java | 262 -
.../coordinator/assign/DefaultAssignerTest.java | 183 -
.../coordinate/BuildJobSubmitterTest.java | 195 -
.../coordinator/coordinate/StreamingTestBase.java | 343 -
stream-core/DEPRECATED_MODULE | 19 -
stream-core/pom.xml | 108 -
.../core/client/HttpReceiverAdminClient.java | 255 -
.../stream/core/client/ReceiverAdminClient.java | 142 -
.../stream/core/consumer/ConsumerStartMode.java | 30 -
.../core/consumer/ConsumerStartProtocol.java | 92 -
.../core/consumer/EndPositionStopCondition.java | 86 -
.../stream/core/consumer/IConsumerProvider.java | 23 -
.../core/consumer/IStopConsumptionCondition.java | 33 -
.../stream/core/consumer/IStreamingConnector.java | 49 -
.../consumer/NeverStopConsumptionCondition.java | 42 -
.../core/consumer/StreamingConsumerChannel.java | 314 -
.../stream/core/dict/LocalDictionaryStore.java | 144 -
.../stream/core/dict/RemoteDictionaryStore.java | 215 -
.../core/dict/StreamingDictionaryClient.java | 216 -
.../core/dict/StreamingDistributedDictionary.java | 91 -
.../core/exception/IllegalStorageException.java | 39 -
.../stream/core/exception/StreamingException.java | 38 -
.../stream/core/metrics/StreamingMetrics.java | 108 -
.../kylin/stream/core/model/AssignRequest.java | 59 -
.../stream/core/model/ConsumerStatsResponse.java | 40 -
.../kylin/stream/core/model/CubeAssignment.java | 147 -
.../kylin/stream/core/model/DataRequest.java | 145 -
.../kylin/stream/core/model/DataResponse.java | 41 -
.../kylin/stream/core/model/HealthCheckInfo.java | 35 -
.../org/apache/kylin/stream/core/model/Node.java | 110 -
.../stream/core/model/PauseConsumersRequest.java | 36 -
.../core/model/RemoteStoreCompleteRequest.java | 58 -
.../apache/kylin/stream/core/model/ReplicaSet.java | 104 -
.../core/model/ReplicaSetLeaderChangeRequest.java | 45 -
.../stream/core/model/ResumeConsumerRequest.java | 45 -
.../kylin/stream/core/model/SegmentBuildState.java | 144 -
.../stream/core/model/StartConsumersRequest.java | 47 -
.../stream/core/model/StopConsumersRequest.java | 45 -
.../core/model/StreamingCubeConsumeState.java | 23 -
.../kylin/stream/core/model/StreamingMessage.java | 79 -
.../kylin/stream/core/model/UnAssignRequest.java | 36 -
.../stream/core/model/stats/ClusterState.java | 83 -
.../stream/core/model/stats/ConsumerStats.java | 105 -
.../stream/core/model/stats/CubeRealTimeState.java | 52 -
.../stream/core/model/stats/LongLatencyInfo.java | 85 -
.../core/model/stats/PartitionConsumeStats.java | 86 -
.../model/stats/ReceiverCubeRealTimeState.java | 47 -
.../stream/core/model/stats/ReceiverCubeStats.java | 100 -
.../stream/core/model/stats/ReceiverState.java | 82 -
.../stream/core/model/stats/ReceiverStats.java | 80 -
.../stream/core/model/stats/ReplicaSetState.java | 95 -
.../stream/core/model/stats/SegmentStats.java | 91 -
.../stream/core/model/stats/SegmentStoreStats.java | 47 -
.../stream/core/query/HavingFilterChecker.java | 135 -
.../stream/core/query/IStreamingGTSearcher.java | 25 -
.../stream/core/query/IStreamingSearchResult.java | 59 -
.../core/query/MultiThreadsResultCollector.java | 161 -
.../kylin/stream/core/query/RecordsAggregator.java | 160 -
.../stream/core/query/ResponseResultSchema.java | 157 -
.../kylin/stream/core/query/ResultCollector.java | 76 -
.../core/query/SingleThreadResultCollector.java | 75 -
.../query/StreamingBuiltInFunctionTransformer.java | 272 -
.../core/query/StreamingCubeDataSearcher.java | 196 -
.../core/query/StreamingDataQueryPlanner.java | 125 -
.../stream/core/query/StreamingQueryProfile.java | 195 -
.../stream/core/query/StreamingSearchContext.java | 161 -
.../stream/core/query/StreamingTupleConverter.java | 144 -
.../stream/core/query/StreamingTupleIterator.java | 126 -
.../kylin/stream/core/source/ISourcePosition.java | 46 -
.../stream/core/source/ISourcePositionHandler.java | 38 -
.../core/source/IStreamingMessageParser.java | 28 -
.../kylin/stream/core/source/IStreamingSource.java | 61 -
.../stream/core/source/MessageFormatException.java | 37 -
.../stream/core/source/MessageParserInfo.java | 83 -
.../apache/kylin/stream/core/source/Partition.java | 96 -
.../stream/core/source/StreamingSourceConfig.java | 99 -
.../core/source/StreamingSourceConfigManager.java | 185 -
.../stream/core/source/StreamingSourceFactory.java | 44 -
.../core/source/StreamingTableSourceInfo.java | 49 -
.../kylin/stream/core/storage/CheckPoint.java | 135 -
.../kylin/stream/core/storage/CheckPointStore.java | 225 -
.../core/storage/IStreamingSegmentStore.java | 62 -
.../apache/kylin/stream/core/storage/Record.java | 68 -
.../stream/core/storage/StreamingCubeSegment.java | 181 -
.../core/storage/StreamingSegmentManager.java | 673 -
.../core/storage/columnar/ColumnDataReader.java | 28 -
.../core/storage/columnar/ColumnDataWriter.java | 27 -
.../columnar/ColumnarMemoryStorePersister.java | 408 -
.../storage/columnar/ColumnarMetricsEncoding.java | 37 -
.../columnar/ColumnarMetricsEncodingFactory.java | 181 -
.../core/storage/columnar/ColumnarRecordCodec.java | 105 -
.../storage/columnar/ColumnarSegmentStore.java | 477 -
.../ColumnarSegmentStoreFilesSearcher.java | 104 -
.../core/storage/columnar/ColumnarStoreCache.java | 182 -
.../storage/columnar/ColumnarStoreCacheStats.java | 95 -
.../storage/columnar/ColumnarStoreDimDesc.java | 95 -
.../storage/columnar/ColumnarStoreMetricsDesc.java | 90 -
.../core/storage/columnar/DataSegmentFragment.java | 159 -
.../columnar/FSInputGeneralColumnDataReader.java | 80 -
.../storage/columnar/FragmentCuboidReader.java | 177 -
.../stream/core/storage/columnar/FragmentData.java | 111 -
.../storage/columnar/FragmentFileSearcher.java | 369 -
.../core/storage/columnar/FragmentFilesMerger.java | 656 -
.../stream/core/storage/columnar/FragmentId.java | 89 -
.../storage/columnar/FragmentSearchResult.java | 491 -
.../storage/columnar/FragmentsMergeResult.java | 53 -
.../storage/columnar/GeneralColumnDataReader.java | 106 -
.../storage/columnar/GeneralColumnDataWriter.java | 60 -
.../storage/columnar/ParsedStreamingCubeInfo.java | 263 -
.../stream/core/storage/columnar/RawRecord.java | 91 -
.../core/storage/columnar/SegmentMemoryStore.java | 444 -
.../storage/columnar/StringArrayComparator.java | 50 -
.../columnar/TimeDerivedColumnEncoding.java | 42 -
.../storage/columnar/compress/Compression.java | 23 -
.../compress/FSInputLZ4CompressedColumnReader.java | 121 -
.../compress/FSInputNoCompressedColumnReader.java | 85 -
.../compress/FSInputRLECompressedColumnReader.java | 130 -
.../compress/LZ4CompressedColumnReader.java | 125 -
.../compress/LZ4CompressedColumnWriter.java | 78 -
.../columnar/compress/LZ4CompressorTest.java | 66 -
.../compress/NoCompressedColumnReader.java | 86 -
.../compress/NoCompressedColumnWriter.java | 40 -
.../compress/RunLengthCompressedColumnReader.java | 160 -
.../compress/RunLengthCompressedColumnWriter.java | 130 -
.../invertindex/ColInvertIndexSearcher.java | 169 -
.../columnar/invertindex/ColInvertIndexWriter.java | 43 -
.../invertindex/FixLenColInvertIndexWriter.java | 87 -
.../invertindex/FixLenIIColumnDescriptor.java | 33 -
.../columnar/invertindex/IIColumnDescriptor.java | 33 -
.../columnar/invertindex/IndexSearchResult.java | 30 -
.../columnar/invertindex/InvertIndexSearcher.java | 341 -
.../invertindex/SeqColInvertIndexWriter.java | 111 -
.../invertindex/SeqIIColumnDescriptor.java | 33 -
.../storage/columnar/protocol/CuboidMetaInfo.java | 114 -
.../columnar/protocol/DimDictionaryMetaInfo.java | 93 -
.../columnar/protocol/DimensionMetaInfo.java | 140 -
.../core/storage/columnar/protocol/Footer.java | 87 -
.../columnar/protocol/FragmentMetaInfo.java | 171 -
.../storage/columnar/protocol/MetricMetaInfo.java | 142 -
.../core/storage/rocksdb/RocksDBSegmentStore.java | 136 -
.../core/util/CompareFilterTimeRangeChecker.java | 145 -
.../apache/kylin/stream/core/util/Constants.java | 24 -
.../apache/kylin/stream/core/util/HDFSUtil.java | 86 -
.../kylin/stream/core/util/NamedThreadFactory.java | 45 -
.../apache/kylin/stream/core/util/NodeUtil.java | 72 -
.../kylin/stream/core/util/RecordsSerializer.java | 127 -
.../apache/kylin/stream/core/util/RestService.java | 133 -
.../kylin/stream/core/util/RetryCallable.java | 35 -
.../apache/kylin/stream/core/util/RetryCaller.java | 83 -
.../kylin/stream/core/util/StreamFilterUtil.java | 56 -
.../stream/core/storage/CheckPointStoreTest.java | 153 -
.../stream/core/storage/MockPositionHandler.java | 72 -
.../stream/core/storage/RecordsSerDeTest.java | 91 -
.../core/storage/StreamingSegmentManagerTest.java | 229 -
.../kylin/stream/core/storage/TestHelper.java | 158 -
.../storage/columnar/ColumnarSegmentStoreTest.java | 206 -
.../storage/columnar/FragmentCuboidReaderTest.java | 134 -
.../storage/columnar/FragmentFileSearcherTest.java | 208 -
.../storage/columnar/FragmentFilesMergerTest.java | 229 -
.../storage/columnar/GeneralColumnDataTest.java | 89 -
.../storage/columnar/SegmentMemoryStoreTest.java | 252 -
.../storage/columnar/StreamingDataSimulator.java | 134 -
.../columnar/compress/LZ4CompressColumnTest.java | 105 -
.../columnar/compress/NoCompressColumnTest.java | 98 -
.../compress/RunLengthCompressColumnTest.java | 166 -
.../ColInvertIndexWriterWriterTest.java | 79 -
.../SearchableColInvertIndexWriterTest.java | 40 -
.../SimpleColInvertIndexWriterTest.java | 72 -
.../performance/FragmentCuboidReaderPerfTest.java | 205 -
.../columnar/performance/PerfDataPrepare.java | 86 -
.../columnar/performance/PerformanceTest.java | 165 -
.../StreamingCubeDataSearcherPerfTest.java | 187 -
.../kylin/stream/core/util/DataGenerator.java | 79 -
stream-receiver/DEPRECATED_MODULE | 19 -
stream-receiver/pom.xml | 305 -
.../stream/server/ReplicaSetLeaderSelector.java | 100 -
.../apache/kylin/stream/server/ServerContext.java | 43 -
.../kylin/stream/server/StreamingReceiver.java | 147 -
.../kylin/stream/server/StreamingServer.java | 777 -
.../server/rest/controller/AdminController.java | 171 -
.../server/rest/controller/BasicController.java | 69 -
.../server/rest/controller/DataController.java | 171 -
.../server/rest/controller/QueryController.java | 104 -
.../server/rest/controller/StatsController.java | 68 -
.../server/rest/controller/SystemController.java | 135 -
.../server/rest/exception/BadRequestException.java | 61 -
.../server/rest/exception/ForbiddenException.java | 41 -
.../rest/exception/InternalErrorException.java | 63 -
.../server/rest/exception/NotFoundException.java | 35 -
.../stream/server/rest/model/ErrorResponse.java | 38 -
.../server/rest/model/PrepareSqlRequest.java | 121 -
.../kylin/stream/server/rest/model/SQLRequest.java | 132 -
.../stream/server/rest/model/SQLResponse.java | 167 -
.../rest/security/StreamTableInterceptor.java | 64 -
.../stream/server/rest/service/BasicService.java | 55 -
.../stream/server/rest/service/QueryService.java | 274 -
.../kylin/stream/server/rest/util/QueryUtil.java | 224 -
.../server/retention/RetentionPolicyInfo.java | 60 -
.../stream/server/storage/LocalStreamStorage.java | 38 -
.../server/storage/LocalStreamStorageQuery.java | 64 -
.../src/main/resources/applicationContext.xml | 77 -
.../src/main/resources/ehcache-test.xml | 37 -
stream-receiver/src/main/resources/ehcache.xml | 37 -
.../resources/stream-receiver-log4j.properties | 44 -
.../src/main/webapp/WEB-INF/kylin-servlet.xml | 19 -
stream-receiver/src/main/webapp/index.html | 23 -
stream-source-kafka/DEPRECATED_MODULE | 19 -
stream-source-kafka/pom.xml | 81 -
.../stream/source/kafka/AbstractTimeParser.java | 38 -
.../stream/source/kafka/BootstrapServerConfig.java | 54 -
.../kylin/stream/source/kafka/DateTimeParser.java | 55 -
.../source/kafka/KafkaBatchSourceAdaptor.java | 43 -
.../source/kafka/KafkaConsumerStartInfo.java | 43 -
.../kylin/stream/source/kafka/KafkaPosition.java | 111 -
.../stream/source/kafka/KafkaPositionHandler.java | 86 -
.../kylin/stream/source/kafka/KafkaSource.java | 334 -
.../stream/source/kafka/KafkaTopicAssignment.java | 64 -
.../kylin/stream/source/kafka/LongTimeParser.java | 63 -
.../stream/source/kafka/TimedJsonStreamParser.java | 204 -
.../source/kafka/consumer/KafkaConnector.java | 125 -
.../assign/KafkaSourcePositionHandlerTest.java | 70 -
.../kylin/stream/source/kafka/KafkaSourceTest.java | 58 -
.../source/kafka/KafkaTopicAssignmentTest.java | 108 -
.../source/kafka/TimedJsonStreamParserTest.java | 166 -
.../src/test/resources/message.json | 51 -
1399 files changed, 18 insertions(+), 796941 deletions(-)
delete mode 100644 core-dictionary/.settings/org.eclipse.core.resources.prefs
delete mode 100644 core-dictionary/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 core-dictionary/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 core-dictionary/DEPRECATED_MODULE
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/ByteComparator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/BytesConverter.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
delete mode 100755 core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueSortedEnumerator.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictNode.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSlice.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendDictSliceKey.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ILookupTable.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupProviderFactory.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
delete mode 100644 core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTableSerializer.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
delete mode 100755 core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
delete mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/lookup/SnapshotManagerTest.java
delete mode 100644 core-dictionary/src/test/resources/dict/dw_category_grouping_names.dat
delete mode 100644 core-dictionary/src/test/resources/dict/english-words.80 (scowl-2015.05.18).txt
delete mode 100644 datasource-sdk/DEPRECATED_MODULE
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AbstractJdbcAdaptor.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AdaptorConfig.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/H2Adaptor.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/MysqlAdaptor.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/PostgresqlAdaptor.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/AdaptorFactory.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/FixedCachedRowSetImpl.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/JdbcConnector.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/SourceConnectorFactory.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvMaster.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvSqlWriter.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/DefaultConfigurer.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/GenericSqlConverter.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParser.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverter.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlNodeConverter.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlParamsFinder.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/DataSourceDef.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/DataSourceDefProvider.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/FunctionDef.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/PropertyDef.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/TypeDef.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/def/TypeMapping.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/utils/ExpressionComparator.java
delete mode 100644 datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/utils/XmlUtil.java
delete mode 100644 datasource-sdk/src/main/resources/datasource/default.xml
delete mode 100644 datasource-sdk/src/main/resources/datasource/mysql.xml
delete mode 100644 datasource-sdk/src/main/resources/datasource/postgresql.xml
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/adaptor/AdaptorConfigTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/JdbcConnectorTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/SourceConnectorFactoryTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParserTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/conv/PostgresqlSqlConverterTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverterTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/conv/SqlParamsFinderTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/def/DataSourceDefTest.java
delete mode 100644 datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/framework/def/TypeDefTest.java
delete mode 100644 datasource-sdk/src/test/resources/datasource/default.xml
delete mode 100644 datasource-sdk/src/test/resources/datasource/default.xml.override
delete mode 100644 datasource-sdk/src/test/resources/datasource/testing.xml
delete mode 100644 engine-flink/DEPRECATED_MODULE
delete mode 100644 engine-flink/pom.xml
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkBatchCubingEngine2.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkBatchCubingJobBuilder2.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkBatchMergeJobBuilder2.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkCubingByLayer.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkCubingMerge.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkExecutable.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkMergingDictionary.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkOnYarnConfigMapping.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/FlinkUtil.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/IFlinkInput.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/IFlinkOutput.java
delete mode 100644 engine-flink/src/main/java/org/apache/kylin/engine/flink/util/PercentileCounterSerializer.java
delete mode 100644 engine-flink/src/test/java/org/apache/kylin/engine/flink/FlinkOnYarnConfigMappingTest.java
delete mode 100644 engine-mr/.settings/org.eclipse.core.resources.prefs
delete mode 100644 engine-mr/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 engine-mr/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 engine-mr/DEPRECATED_MODULE
delete mode 100644 engine-mr/pom.xml
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobBuilder2.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchOptimizeJobCheckpointBuilder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/IInput.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/ILookupMaterializer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/LookupMaterializeContext.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/LookupSnapshotBuildJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/LookupSnapshotJobBuilder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFileReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingEngine.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/ConvergeCuboidDataUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeJobLockUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidSchedulerUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultX509TrustManager.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DictionaryGetterUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobRelatedMetaUtil.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/SerializableConfiguration.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/exception/HadoopShellException.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/exception/MapReduceException.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/exception/SegmentNotFoundException.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidPartitioner.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ConvergeCuboidDataPartitioner.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ConvergeCuboidDataReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CopyDictionaryStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExtractDictionaryFromGlobalJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExtractDictionaryFromGlobalMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerMapping.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidFromBaseCuboidReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapperBase.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
delete mode 100755 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SegmentReEncoder.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UHCDictionaryJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UHCDictionaryMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UHCDictionaryPartitioner.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UHCDictionaryReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterCheckpointStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/filter/UHCDictPathFilter.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupExecutableUtil.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupSnapshotToMetaStoreStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/UpdateCubeAfterSnapshotStep.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarFilesReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDataInputFormat.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDataReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDictInputFormat.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDictReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitInputFormat.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/DictsReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictJob.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictMapper.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictReducer.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/RowRecord.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/RowRecordReader.java
delete mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/SaveDictStep.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/common/DefaultX509TrustManagerTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/common/HadoopCmdOutputTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/common/HadoopJobStatusCheckerTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/common/JobInfoConverterTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/common/MapReduceExecutableTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/DictColDeduperTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerMappingTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/LookupMaterializeContextTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
delete mode 100644 engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/lookup/LookupExecutableUtilTest.java
delete mode 100755 engine-mr/src/test/resources/data/8d_cuboid/part-r-00000
delete mode 100755 engine-mr/src/test/resources/data/base_cuboid/part-r-00000
delete mode 100644 engine-mr/src/test/resources/dict/DW_SITES
delete mode 100644 engine-mr/src/test/resources/multi_file_double/data_0
delete mode 100644 engine-mr/src/test/resources/multi_file_double/data_1
delete mode 100644 engine-mr/src/test/resources/multi_file_double/data_2
delete mode 100644 engine-mr/src/test/resources/multi_file_double/data_3
delete mode 100644 engine-mr/src/test/resources/multi_file_double/data_4
delete mode 100644 engine-mr/src/test/resources/multi_file_empty_file/empty.txt
delete mode 100644 engine-mr/src/test/resources/multi_file_empty_file/has_value.txt
delete mode 100644 engine-mr/src/test/resources/multi_file_int/data_0
delete mode 100644 engine-mr/src/test/resources/multi_file_int/data_1
delete mode 100644 engine-mr/src/test/resources/multi_file_int/data_2
delete mode 100644 engine-mr/src/test/resources/multi_file_int/data_3
delete mode 100644 engine-mr/src/test/resources/multi_file_int/data_4
delete mode 100644 engine-mr/src/test/resources/multi_file_str/data_0
delete mode 100644 engine-mr/src/test/resources/multi_file_str/data_1
delete mode 100644 engine-mr/src/test/resources/multi_file_str/data_2
delete mode 100644 engine-mr/src/test/resources/multi_file_str/data_3
delete mode 100644 engine-mr/src/test/resources/multi_file_str/data_4
delete mode 100644 engine-spark/.settings/org.eclipse.core.resources.prefs
delete mode 100644 engine-spark/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 engine-spark/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 engine-spark/DEPRECATED_MODULE
delete mode 100644 engine-spark/pom.xml
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/ISparkInput.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/ISparkOutput.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinSparkJobListener.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/MultipleOutputsRDD.scala
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchMergeJobBuilder2.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBuildDictionary.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkColumnCardinality.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCreatingFlatTable.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingMerge.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkMergingDictionary.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkSqlBatch.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkSqlOnLivyBatch.scala
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUtil.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/exception/SparkException.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
delete mode 100644 engine-spark/src/main/java/org/apache/kylin/engine/spark/util/PercentileCounterSerializer.java
delete mode 100644 engine-spark/src/test/java/org/apache/kylin/engine/spark/util/IteratorUtilsTest.java
delete mode 100644 engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
delete mode 100644 engine-spark/src/test/java/org/apache/kylin/engine/spark/util/PercentileSerializerTest.java
delete mode 100644 kylin-it/.settings/org.eclipse.core.resources.prefs
delete mode 100644 kylin-it/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 kylin-it/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 kylin-it/DEPRECATED_MODULE
delete mode 100644 kylin-it/pom.xml
delete mode 100755 kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
delete mode 100755 kylin-it/src/test/java/org/apache/kylin/cube/cuboid/algorithm/ITAlgorithmTestBase.java
delete mode 100755 kylin-it/src/test/java/org/apache/kylin/cube/cuboid/algorithm/ITGeneticAlgorithmTest.java
delete mode 100755 kylin-it/src/test/java/org/apache/kylin/cube/cuboid/algorithm/ITGreedyAlgorithmTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/dict/ITGlobalDictionaryBuilderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/job/ITDistributedSchedulerBaseTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/job/ITDistributedSchedulerTakeOverTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/provision/NetworkUtils.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/CompareQueryBySuffix.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/ICompareQueryTranslator.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/realtime/BuildCubeWithStreamV2.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/SourceManagerTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
delete mode 100644 kylin-it/src/test/java/org/apache/kylin/storage/jdbc/ITJDBCResourceStoreTest.java
delete mode 100644 kylin-it/src/test/resources/logging.properties
delete mode 100644 kylin-it/src/test/resources/query/sql/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query100.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query101.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query102.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query103.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query104.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query105.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query106.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query107.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query108.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query109.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query110.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query112.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query113.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query114.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query115.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query14.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query15.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query16.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query17.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query18.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query19.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query20.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query21.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query22.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query23.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query24.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query25.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query26.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query27.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query28.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query29.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query30.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query31.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query32.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query33.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query34.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query35.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query36.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query37.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query38.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query39.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query40.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query41.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query42.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query43.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query44.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query49.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query50.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query51.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query52.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query53.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query54.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query55.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query56.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query57.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query58.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query59.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query60.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query61.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query62.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query63.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query64.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query65.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query66.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query67.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query68.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query69.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query70.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query71.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query72.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query73.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query74.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query75.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query76.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query77.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query78.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query79.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query80.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query83.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query84.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query85.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query86.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query87.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query88.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query89.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query90.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query91.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query92.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query93.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query94.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query95.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query96.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query97.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query98.sql
delete mode 100644 kylin-it/src/test/resources/query/sql/query99.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_cache/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query55.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query56.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query57.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_casewhen/query58.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_castprunesegs/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_castprunesegs/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_castprunesegs/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_datetime/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query11.sql
delete mode 100755 kylin-it/src/test/resources/query/sql_derived/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_derived/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dict_enumerator/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dict_enumerator/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dict_enumerator/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query00.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query00.sql.expected/.part-00000-a34638f6-3788-46e2-b662-1868271164b4-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query00.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query00.sql.expected/part-00000-a34638f6-3788-46e2-b662-1868271164b4-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query01.sql.expected/.part-00000-21219da1-c7c9-4934-8c6f-82b434e27fac-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query01.sql.expected/part-00000-21219da1-c7c9-4934-8c6f-82b434e27fac-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query02.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query02.sql.expected/.part-00000-85399230-d604-4ccf-8cf6-899a065f39a2-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query02.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query02.sql.expected/part-00000-85399230-d604-4ccf-8cf6-899a065f39a2-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query03.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query03.sql.expected/.part-00000-49fb3791-2e08-4e43-af9a-2c64d78a3f22-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query03.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query03.sql.expected/part-00000-49fb3791-2e08-4e43-af9a-2c64d78a3f22-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query04.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query04.sql.expected/.part-00000-59c5f310-37b7-44d3-a8ce-461be48775d2-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query04.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query04.sql.expected/part-00000-59c5f310-37b7-44d3-a8ce-461be48775d2-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query05.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query05.sql.expected/.part-00000-697b0e0c-49f7-427d-826d-f2a8bc2c62b6-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query05.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query05.sql.expected/part-00000-697b0e0c-49f7-427d-826d-f2a8bc2c62b6-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query06.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query06.sql.expected/.part-00000-e98c49ba-f80b-4526-bded-1542272f5248-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query06.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query06.sql.expected/part-00000-e98c49ba-f80b-4526-bded-1542272f5248-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query07.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query07.sql.expected/.part-00000-6c1b6cf6-4979-40ea-b24b-6e9bba99fc77-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query07.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct/query07.sql.expected/part-00000-6c1b6cf6-4979-40ea-b24b-6e9bba99fc77-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_dim/query100.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_dim/query101.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely_rollup/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_distinct_precisely_rollup/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query01.dat
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query02.dat
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query03.dat
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query04.dat
delete mode 100644 kylin-it/src/test/resources/query/sql_dynamic/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query00.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query00.sql.expected/.part-00000-ddb834cd-5c9e-4c0b-8074-7238869f186c-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query00.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query00.sql.expected/part-00000-ddb834cd-5c9e-4c0b-8074-7238869f186c-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query01.sql.expected/.part-00000-c43fdd67-7c83-4382-82dd-bb19338fb3f5-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query01.sql.expected/part-00000-c43fdd67-7c83-4382-82dd-bb19338fb3f5-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query02.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query02.sql.expected/.part-00000-aa19cdab-4abc-4173-92d1-8d190775b8bd-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query02.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query02.sql.expected/part-00000-aa19cdab-4abc-4173-92d1-8d190775b8bd-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query03.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query03.sql.expected/.part-00000-5afae054-7f80-461f-ab4a-bc19d5f6bda3-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query03.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query03.sql.expected/part-00000-5afae054-7f80-461f-ab4a-bc19d5f6bda3-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query04.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query04.sql.expected/.part-00000-7f12f067-288d-4fa8-8ded-2613ec384f20-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query04.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query04.sql.expected/part-00000-7f12f067-288d-4fa8-8ded-2613ec384f20-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query05.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query05.sql.expected/.part-00000-17e15acc-11b7-42f9-8faf-913c2d4cbf62-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query05.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query05.sql.expected/part-00000-17e15acc-11b7-42f9-8faf-913c2d4cbf62-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query06.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query06.sql.expected/.part-00000-0220bf7e-f8b3-475f-b50b-09916d61637a-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query06.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query06.sql.expected/part-00000-0220bf7e-f8b3-475f-b50b-09916d61637a-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query07.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query07.sql.expected/.part-00000-29e1e80c-4fef-4263-9331-273675453c90-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query07.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query07.sql.expected/part-00000-29e1e80c-4fef-4263-9331-273675453c90-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query08.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query08.sql.expected/.part-00000-0745836b-bc11-4eac-807d-02c54e8b1337-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query08.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query08.sql.expected/part-00000-0745836b-bc11-4eac-807d-02c54e8b1337-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query09.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query09.sql.expected/.part-00000-ca31db76-924c-4045-bd66-369c632ce1e1-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query09.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query09.sql.expected/part-00000-ca31db76-924c-4045-bd66-369c632ce1e1-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query10.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query10.sql.expected/.part-00000-87c05bff-a47f-46e0-afcc-7bb0652e6880-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query10.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query10.sql.expected/part-00000-87c05bff-a47f-46e0-afcc-7bb0652e6880-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg_multi_segment/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_exactly_agg_multi_segment/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_expression/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_function/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_grouping/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query05.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query05.sql.expected/.part-00000-6ae62711-bb46-4c98-94ad-17ed4ae32386-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query05.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query05.sql.expected/part-00000-6ae62711-bb46-4c98-94ad-17ed4ae32386-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_h2/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_hive/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_hive/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query00.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query00.sql.expected/.part-00000-24e31c51-c35a-4d88-9eea-e3ef825967d8-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query00.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query00.sql.expected/part-00000-24e31c51-c35a-4d88-9eea-e3ef825967d8-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query01.sql.expected/.part-00000-09c77c16-39dd-489d-9078-2a12bb25ef3f-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query01.sql.expected/part-00000-09c77c16-39dd-489d-9078-2a12bb25ef3f-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query03.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query03.sql.expected/.part-00000-0c535c4a-649e-4a27-8ee3-f839e858e96b-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query03.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query03.sql.expected/part-00000-0c535c4a-649e-4a27-8ee3-f839e858e96b-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query04.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query04.sql.expected/.part-00000-0ba75ff1-920d-4161-a3ad-080d3e71ce25-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query04.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query04.sql.expected/part-00000-0ba75ff1-920d-4161-a3ad-080d3e71ce25-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query05.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query05.sql.expected/.part-00000-a3ed9048-d2e2-414a-8588-9ed588204a97-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query05.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_count/query05.sql.expected/part-00000-a3ed9048-d2e2-414a-8588-9ed588204a97-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query00.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query00.sql.expected/.part-00000-3a6bc9a2-0d5a-4839-94e1-d2ec55db7c1b-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query00.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query00.sql.expected/part-00000-3a6bc9a2-0d5a-4839-94e1-d2ec55db7c1b-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query01.sql.expected/.part-00000-513c8db3-f109-43f8-b8bc-c02ff5f0405c-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query01.sql.expected/part-00000-513c8db3-f109-43f8-b8bc-c02ff5f0405c-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query03.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query03.sql.expected/.part-00000-77c6dcd8-9969-4209-bd1e-b5906fe175c6-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query03.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query03.sql.expected/part-00000-77c6dcd8-9969-4209-bd1e-b5906fe175c6-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query04.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query04.sql.expected/.part-00000-892bf65a-af47-44ab-84b2-56abc974a2b0-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query04.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query04.sql.expected/part-00000-892bf65a-af47-44ab-84b2-56abc974a2b0-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query05.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query05.sql.expected/.part-00000-bf3de84f-1d22-4777-8876-908c30a97290-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query05.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query05.sql.expected/part-00000-bf3de84f-1d22-4777-8876-908c30a97290-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query06.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query06.sql.expected/.part-00000-49e47126-09a8-4b4c-b022-a3c537ab5c04-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query06.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_intersect_value/query06.sql.expected/part-00000-49e47126-09a8-4b4c-b022-a3c537ab5c04-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_join/query_00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query15.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query16.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query17.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query18.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query20.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query21.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query22.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_like/query23.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_limit/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_limit/query00.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_limit/query00.sql.expected/.part-00000-a290bf53-2025-46e5-8133-2a6bf503206c-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_limit/query00.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_limit/query00.sql.expected/part-00000-a290bf53-2025-46e5-8133-2a6bf503206c-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query45.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query46.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query47.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query48.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_lookup/query55.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_multi_model/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_non_equal_join/query_00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_non_equal_join/query_01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_non_equal_join/query_02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_orderby/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_orderby/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_orderby/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_ordinal/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_ordinal/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query01.sql.expected/.part-00000-5f74fce0-2455-4b7d-baa1-91c4bed92238-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query01.sql.expected/part-00000-5f74fce0-2455-4b7d-baa1-91c4bed92238-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query02.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query02.sql.expected/.part-00000-435b7736-3444-4bbb-b4e1-accdbc25c404-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query02.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query02.sql.expected/part-00000-435b7736-3444-4bbb-b4e1-accdbc25c404-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_percentile/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query01.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query01.sql.expected/.part-00000-6cd6687e-124c-4090-8cc6-36c9aa61c660-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query01.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query01.sql.expected/part-00000-6cd6687e-124c-4090-8cc6-36c9aa61c660-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query02.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query02.sql.expected/.part-00000-a3d3b74d-27b2-4437-a05a-4b066f84b43e-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query02.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_plan/query02.sql.expected/part-00000-a3d3b74d-27b2-4437-a05a-4b066f84b43e-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query02.sql.expected/._SUCCESS.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query02.sql.expected/.part-00000-45343019-e3fd-4ce5-b509-d744f9ccb327-c000.csv.crc
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query02.sql.expected/_SUCCESS
delete mode 100644 kylin-it/src/test/resources/query/sql_prune_segment/query02.sql.expected/part-00000-45343019-e3fd-4ce5-b509-d744f9ccb327-c000.csv
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_snowflake/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming/query14.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/count/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/not_compare_result/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/not_compare_result/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query14.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query15.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query16.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query17.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query18.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query19.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query21.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query22.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query23.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query24.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query25.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query26.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query27.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query28.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query29.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query30.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query31.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query32.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query33.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query34.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query35.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query36.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_subquery/query37.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query10.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query14.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query15.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query16.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query17.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query18.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query19.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query20.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query21.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query22.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query23.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query24.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query25.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query27.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query28.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query29.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_tableau/query67.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timeout/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query13.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query14.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query21.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query21_a.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22_a.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22_b.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22_c.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22_d.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query22_e.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query23.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query23_a.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query24.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query24_a.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query25.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query25_a.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_timestamp/query26.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_topn/query45.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_topn/query81.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_topn/query82.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_topn/query83.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_union/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_union/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_union/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_unionall/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_unionall/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_unionall/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_values/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_values/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_values/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_values/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_values/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query00.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query01.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query02.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query03.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query04.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query05.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query06.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query07.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query08.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query09.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query11.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query12.sql
delete mode 100644 kylin-it/src/test/resources/query/sql_window/query13.sql
delete mode 100644 kylin-it/src/test/resources/statistics.txt
delete mode 100644 kylin-it/src/test/resources/streaming_v2_user_info_messages.txt
delete mode 100644 kylin-it/src/test/scala/kafka/admin/AdminUtils.scala
delete mode 100644 source-jdbc/DEPRECATED_MODULE
delete mode 100644 source-jdbc/pom.xml
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/HiveCmdStep.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveSparkInput.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcSource.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTable.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTableReader.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcExplorer.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveMRInput.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveSparkInput.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcSource.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcTable.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcTableReader.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadata.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/IJdbcMetadata.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/PostgresqlJdbcMetadata.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadata.java
delete mode 100644 source-jdbc/src/main/java/org/apache/kylin/source/jdbc/sqoop/SqoopCmdStep.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcExplorerTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcHiveInputBaseTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/extensible/JdbcExplorerTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveMRInputTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/extensible/JdbcSourceTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/extensible/JdbcTableTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/extensible/TestBase.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadataTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
delete mode 100644 source-jdbc/src/test/resources/ut_meta/jdbc_source/cube/ut_jdbc_shard.json
delete mode 100644 source-jdbc/src/test/resources/ut_meta/jdbc_source/cube_desc/ut_jdbc_shard.json
delete mode 100644 source-kafka/.settings/org.eclipse.core.resources.prefs
delete mode 100644 source-kafka/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 source-kafka/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 source-kafka/DEPRECATED_MODULE
delete mode 100644 source-kafka/pom.xml
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/AbstractTimeParser.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/DateTimeParser.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/DefaultTimeParser.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaInputBase.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSparkInput.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaInputFormat.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaInputRecordReader.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaInputSplit.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/model/StreamCubeFactTableDesc.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
delete mode 100644 source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
delete mode 100644 source-kafka/src/test/java/org/apache/kylin/source/kafka/TimedJsonStreamParserTest.java
delete mode 100644 source-kafka/src/test/java/org/apache/kylin/source/kafka/config/KafkaConsumerPropertiesTest.java
delete mode 100644 source-kafka/src/test/java/org/apache/kylin/source/kafka/hadoop/KafkaInputRecordReaderTest.java
delete mode 100644 source-kafka/src/test/resources/kylin-kafka-consumer.xml
delete mode 100644 source-kafka/src/test/resources/message.json
delete mode 100644 storage-hbase/.settings/org.eclipse.core.resources.prefs
delete mode 100644 storage-hbase/.settings/org.eclipse.jdt.core.prefs
delete mode 100644 storage-hbase/.settings/org.eclipse.jdt.ui.prefs
delete mode 100644 storage-hbase/DEPRECATED_MODULE
delete mode 100644 storage-hbase/pom.xml
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorFilter.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CellListIterator.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
delete mode 100755 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SegmentQueryCache.java
delete mode 100755 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SegmentQueryResult.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMRSteps.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMaterializer.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupProvider.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupRowEncoder.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupTable.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/KVSortReducerWithDupKeyCheck.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/LookupTableHFilesBulkLoadJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/LookupTableToHFileJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/LookupTableToHFileMapper.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/UpdateSnapshotCacheForQueryServersStep.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseFlinkOutputTransition.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseJobSteps.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkOutputTransition.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseSparkSteps.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowKeyWritable.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SparkCubeHFile.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLIOps.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUnionUtil.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/Results.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
delete mode 100644 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperAclBuilder.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/RowProjectorTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/SegmentQueryResultTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupRowEncoderTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/lookup/LookupTableToHFileJobTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/lookup/UpdateSnapshotCacheForQueryServersStepTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HFilePartitionerTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLITest.java
delete mode 100644 storage-hbase/src/test/java/org/apache/kylin/storage/hbase/util/ZookeeperAclBuilderTest.java
delete mode 100644 storage-hbase/src/test/resources/data/8d_cuboid/part-r-00000
delete mode 100644 storage-hbase/src/test/resources/data/base_cuboid/part-r-00000
delete mode 100644 storage-hbase/src/test/resources/partition_list/_partition.lst
delete mode 100644 storage-hbase/src/test/resources/partition_list/part-r-00000
delete mode 100644 storage-stream/DEPRECATED_MODULE
delete mode 100644 storage-stream/pom.xml
delete mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/StreamStorage.java
delete mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/StreamStorageQuery.java
delete mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/HttpStreamDataSearchClient.java
delete mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/IStreamDataSearchClient.java
delete mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/MockedStreamDataSearchClient.java
delete mode 100644 stream-coordinator/DEPRECATED_MODULE
delete mode 100644 stream-coordinator/pom.xml
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/Coordinator.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/HBaseStreamMetadataStore.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/MockStreamMetadataStore.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamMetadataStore.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamMetadataStoreFactory.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamingCubeInfo.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamingUtils.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/ZookeeperStreamMetadataStore.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/Assigner.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/AssignmentUtil.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/AssignmentsCache.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/CubePartitionRoundRobinAssigner.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/DefaultAssigner.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorClient.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorClientFactory.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorResponse.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/HttpCoordinatorClient.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/ReceiverClusterManager.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/SegmentJobBuildInfo.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NonSideEffect.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicAndNotIdempotent.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicIdempotent.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterDoctor.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterStateChecker.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/ClusterStateException.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/CoordinateException.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/NotLeadCoordinatorException.java
delete mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/StoreException.java
delete mode 100644 stream-coordinator/src/main/resources/log4j.properties
delete mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/CoordinatorTest.java
delete mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/assign/CubePartitionRoundRobinAssignerTest.java
delete mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/assign/DefaultAssignerTest.java
delete mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitterTest.java
delete mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/StreamingTestBase.java
delete mode 100644 stream-core/DEPRECATED_MODULE
delete mode 100644 stream-core/pom.xml
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/client/HttpReceiverAdminClient.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/client/ReceiverAdminClient.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/ConsumerStartMode.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/ConsumerStartProtocol.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/EndPositionStopCondition.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IConsumerProvider.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IStopConsumptionCondition.java
delete mode 100755 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IStreamingConnector.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/NeverStopConsumptionCondition.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/LocalDictionaryStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/RemoteDictionaryStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDictionaryClient.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDistributedDictionary.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/exception/IllegalStorageException.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/exception/StreamingException.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/metrics/StreamingMetrics.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/AssignRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/ConsumerStatsResponse.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/CubeAssignment.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/DataRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/DataResponse.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/HealthCheckInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/Node.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/PauseConsumersRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/RemoteStoreCompleteRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/ReplicaSet.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/ReplicaSetLeaderChangeRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/ResumeConsumerRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/SegmentBuildState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/StartConsumersRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/StopConsumersRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/StreamingCubeConsumeState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/StreamingMessage.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/UnAssignRequest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ClusterState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ConsumerStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/CubeRealTimeState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/LongLatencyInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/PartitionConsumeStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverCubeRealTimeState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverCubeStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReplicaSetState.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/SegmentStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/SegmentStoreStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/HavingFilterChecker.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/IStreamingGTSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/IStreamingSearchResult.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/MultiThreadsResultCollector.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/RecordsAggregator.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/ResponseResultSchema.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/ResultCollector.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/SingleThreadResultCollector.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingBuiltInFunctionTransformer.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingCubeDataSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingDataQueryPlanner.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingQueryProfile.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingSearchContext.java
delete mode 100755 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingTupleConverter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingTupleIterator.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/ISourcePosition.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/ISourcePositionHandler.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/IStreamingMessageParser.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/IStreamingSource.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/MessageFormatException.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/MessageParserInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/Partition.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceConfig.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceConfigManager.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceFactory.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingTableSourceInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/CheckPoint.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/CheckPointStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/IStreamingSegmentStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/Record.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/StreamingCubeSegment.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/StreamingSegmentManager.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnDataReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnDataWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMemoryStorePersister.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMetricsEncoding.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMetricsEncodingFactory.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarRecordCodec.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStoreFilesSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreCache.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreCacheStats.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreDimDesc.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreMetricsDesc.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/DataSegmentFragment.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FSInputGeneralColumnDataReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentCuboidReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentData.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFileSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentId.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentSearchResult.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentsMergeResult.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ParsedStreamingCubeInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/RawRecord.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/SegmentMemoryStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/StringArrayComparator.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/TimeDerivedColumnEncoding.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/Compression.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputLZ4CompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputNoCompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputRLECompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressedColumnWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressorTest.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressedColumnWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressedColumnReader.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressedColumnWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/FixLenColInvertIndexWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/FixLenIIColumnDescriptor.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/IIColumnDescriptor.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/IndexSearchResult.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/InvertIndexSearcher.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SeqColInvertIndexWriter.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SeqIIColumnDescriptor.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/CuboidMetaInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/DimDictionaryMetaInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/DimensionMetaInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/Footer.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/FragmentMetaInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/MetricMetaInfo.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/rocksdb/RocksDBSegmentStore.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/CompareFilterTimeRangeChecker.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/Constants.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/HDFSUtil.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/NamedThreadFactory.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/NodeUtil.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RecordsSerializer.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RestService.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RetryCallable.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RetryCaller.java
delete mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/StreamFilterUtil.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/CheckPointStoreTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/MockPositionHandler.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/RecordsSerDeTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/StreamingSegmentManagerTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/TestHelper.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStoreTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentCuboidReaderTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentFileSearcherTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMergerTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/SegmentMemoryStoreTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/StreamingDataSimulator.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressColumnTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressColumnTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressColumnTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexWriterWriterTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SearchableColInvertIndexWriterTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SimpleColInvertIndexWriterTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/FragmentCuboidReaderPerfTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/PerfDataPrepare.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/PerformanceTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/StreamingCubeDataSearcherPerfTest.java
delete mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/util/DataGenerator.java
delete mode 100644 stream-receiver/DEPRECATED_MODULE
delete mode 100644 stream-receiver/pom.xml
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/ReplicaSetLeaderSelector.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/ServerContext.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/StreamingReceiver.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/StreamingServer.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/AdminController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/BasicController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/DataController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/QueryController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/StatsController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/SystemController.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/BadRequestException.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/ForbiddenException.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/InternalErrorException.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/NotFoundException.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/ErrorResponse.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/PrepareSqlRequest.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/SQLRequest.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/SQLResponse.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/security/StreamTableInterceptor.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/service/BasicService.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/service/QueryService.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/util/QueryUtil.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/retention/RetentionPolicyInfo.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/storage/LocalStreamStorage.java
delete mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/storage/LocalStreamStorageQuery.java
delete mode 100644 stream-receiver/src/main/resources/applicationContext.xml
delete mode 100644 stream-receiver/src/main/resources/ehcache-test.xml
delete mode 100644 stream-receiver/src/main/resources/ehcache.xml
delete mode 100644 stream-receiver/src/main/resources/stream-receiver-log4j.properties
delete mode 100644 stream-receiver/src/main/webapp/WEB-INF/kylin-servlet.xml
delete mode 100644 stream-receiver/src/main/webapp/index.html
delete mode 100644 stream-source-kafka/DEPRECATED_MODULE
delete mode 100644 stream-source-kafka/pom.xml
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/AbstractTimeParser.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/BootstrapServerConfig.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/DateTimeParser.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaBatchSourceAdaptor.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaConsumerStartInfo.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaPosition.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaPositionHandler.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaSource.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaTopicAssignment.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/LongTimeParser.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/TimedJsonStreamParser.java
delete mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/consumer/KafkaConnector.java
delete mode 100644 stream-source-kafka/src/test/java/org/apache/kylin/stream/coordinator/assign/KafkaSourcePositionHandlerTest.java
delete mode 100644 stream-source-kafka/src/test/java/org/apache/kylin/stream/source/kafka/KafkaSourceTest.java
delete mode 100644 stream-source-kafka/src/test/java/org/apache/kylin/stream/source/kafka/KafkaTopicAssignmentTest.java
delete mode 100644 stream-source-kafka/src/test/java/org/apache/kylin/stream/source/kafka/TimedJsonStreamParserTest.java
delete mode 100644 stream-source-kafka/src/test/resources/message.json
[kylin] 01/03: Support soft affinity and local cache feature
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch kylin-soft-affinity-local-cache
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0f5b404a68d77190c7db532a7ead5c9216b7f562
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Thu Sep 16 09:03:53 2021 +0800
Support soft affinity and local cache feature
1. Implement LocalDataCacheManager
2. base xiaoxiang's PR
3. Implement CacheFileScanRDD
4. Implement AbstractCacheFileSystem
5. Optimize performance
6. Support soft affinity for hdfs
7. Support ByteBuffer to read data, and avoid to read data one byte by one byte
8. Support to cache small files in memory : ByteBufferPageStore extends PageStore to support cache data in memory
---
.../test/resources/query/test_query/query01.sql | 30 ++
.../pom.xml | 44 +--
.../apache/kylin/cache/KylinCacheConstants.java | 29 ++
.../kylin/cache/fs/AbstractCacheFileSystem.java | 330 +++++++++++++++++++++
.../kylin/cache/fs/AlluxioHdfsFileInputStream.java | 223 ++++++++++++++
.../kylin/cache/fs/CacheFileInputStream.java | 312 +++++++++++++++++++
.../kylin/cache/fs/CacheFileSystemConstants.java | 54 ++++
.../kylin/cache/fs/MemCacheFileInputStream.java | 216 ++++++++++++++
.../kylin/cache/fs/OnlyForTestCacheFileSystem.java | 30 ++
.../kylin/cache/fs/kylin/CacheAllFileSystem.java | 32 ++
.../kylin/cache/fs/kylin/KylinCacheFileSystem.java | 47 +++
.../apache/kylin/cache/utils/ConsistentHash.java | 87 ++++++
.../apache/kylin/cache/utils/ReflectionUtil.java | 168 +++++++++++
.../kylin/softaffinity/SoftAffinityConstants.java | 41 +++
.../kylin/softaffinity/SoftAffinityManager.scala | 146 +++++++++
.../scheduler/SoftAffinityListener.scala | 38 +++
.../strategy/SoftAffinityAllocationTrait.scala | 38 +++
.../strategy/SoftAffinityStrategy.scala | 55 ++++
.../execution/datasources/CacheFilePartition.scala | 72 +++++
.../execution/datasources/CacheFileScanRDD.scala | 77 +++++
kylin-spark-project/kylin-spark-common/pom.xml | 5 +
.../common/logging/AbstractHdfsLogAppender.java | 3 +-
.../common/logging/SparkDriverHdfsLogAppender.java | 3 +-
.../common/logging/SparkExecutorHdfsAppender.java | 4 +-
.../org/apache/spark/utils/SparkHadoopUtils.scala | 30 ++
.../sql/execution/KylinFileSourceScanExec.scala | 19 +-
kylin-spark-project/pom.xml | 1 +
27 files changed, 2107 insertions(+), 27 deletions(-)
diff --git a/kylin-it/src/test/resources/query/test_query/query01.sql b/kylin-it/src/test/resources/query/test_query/query01.sql
new file mode 100644
index 0000000000..d4dec60406
--- /dev/null
+++ b/kylin-it/src/test/resources/query/test_query/query01.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT test_kylin_fact.cal_dt,cast(timestampdiff(DAY,date'2013-01-01',test_kylin_fact.cal_dt) as integer) as x,sum(price) as y
+ FROM TEST_KYLIN_FACT
+
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ GROUP BY test_kylin_fact.cal_dt
+ ORDER BY test_kylin_fact.cal_dt
+;{"scanRowCount":1462,"scanBytes":215217,"scanFiles":2,"cuboidId":262144}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-spark-common/pom.xml b/kylin-spark-project/kylin-soft-affinity-cache/pom.xml
similarity index 80%
copy from kylin-spark-project/kylin-spark-common/pom.xml
copy to kylin-spark-project/kylin-soft-affinity-cache/pom.xml
index 4d0cee3851..87370db8d4 100644
--- a/kylin-spark-project/kylin-spark-common/pom.xml
+++ b/kylin-spark-project/kylin-soft-affinity-cache/pom.xml
@@ -20,37 +20,39 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <name>Apache Kylin 4.X - Common</name>
+ <artifactId>kylin-soft-affinity-cache</artifactId>
<packaging>jar</packaging>
- <artifactId>kylin-spark-common</artifactId>
- <version>4.0.2-SNAPSHOT</version>
+ <name>Apache Kylin 4.X - Soft Affinity and Cache</name>
<parent>
- <artifactId>kylin-spark-project</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>4.0.2-SNAPSHOT</version>
+ <artifactId>kylin-spark-project</artifactId>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <alluxio.version>2.6.1</alluxio.version>
+ </properties>
+
<dependencies>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-spark-metadata</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.alluxio</groupId>
+ <artifactId>alluxio-shaded-client</artifactId>
+ <version>${alluxio.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
@@ -110,4 +112,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/KylinCacheConstants.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/KylinCacheConstants.java
new file mode 100644
index 0000000000..b1232aee0f
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/KylinCacheConstants.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache;
+
+public class KylinCacheConstants {
+
+ private KylinCacheConstants() {
+ }
+
+ // Todo: change the param key name
+ public static final String KYLIN_CACHE_FS =
+ "org.apache.kylin.cache.fs.kylin.KylinCacheFileSystem";
+}
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
new file mode 100644
index 0000000000..7a713194d7
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import alluxio.client.file.CacheContext;
+import alluxio.client.file.URIStatus;
+import alluxio.client.file.cache.CacheManager;
+import alluxio.client.file.cache.LocalCacheFileInStream;
+import alluxio.conf.AlluxioConfiguration;
+import alluxio.hadoop.AlluxioHdfsInputStream;
+import alluxio.hadoop.HadoopFileOpener;
+import alluxio.hadoop.HadoopUtils;
+import alluxio.metrics.MetricsConfig;
+import alluxio.metrics.MetricsSystem;
+import alluxio.wire.FileInfo;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+//import org.apache.hadoop.util.DirectBufferPool;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.cache.utils.ReflectionUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.hash.Hashing.md5;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public abstract class AbstractCacheFileSystem extends FilterFileSystem {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractCacheFileSystem.class);
+
+ //private static final DirectBufferPool bufferPool = new DirectBufferPool();
+
+ protected URI uri;
+ protected String originalScheme;
+ protected int bufferSize = 4096;
+ protected boolean useLocalCache = false;
+ protected boolean useLegacyFileInputStream = false;
+ protected HadoopFileOpener mHadoopFileOpener;
+ protected LocalCacheFileInStream.FileInStreamOpener mAlluxioFileOpener;
+ protected CacheManager mCacheManager;
+ protected AlluxioConfiguration mAlluxioConf;
+
+ protected LoadingCache<Path, FileStatus> fileStatusCache;
+
+ // put("s3", "com.amazon.ws.emr.hadoop.fs.EmrFileSystem");
+ // put("s3n", "com.amazon.ws.emr.hadoop.fs.EmrFileSystem");
+ // put("s3bfs", "org.apache.hadoop.fs.s3.S3FileSystem");
+ protected static final Map<String, String> schemeClassMap = new HashMap<String, String>() {
+ {
+ put("file", "org.apache.hadoop.fs.LocalFileSystem");
+ put("viewfs", "org.apache.hadoop.fs.viewfs.ViewFileSystem");
+ put("s3a", "org.apache.hadoop.fs.s3a.S3AFileSystem");
+ put("s3", "org.apache.hadoop.fs.s3.S3FileSystem");
+ put("s3n", "org.apache.hadoop.fs.s3native.NativeS3FileSystem");
+ put("hdfs", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ put("wasb", "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
+ put("wasbs", "org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure");
+ put("jfs", "io.juicefs.JuiceFileSystem");
+ put("alluxio", "alluxio.hadoop.FileSystem");
+ }
+ };
+
+ /**
+ * Create internal FileSystem
+ */
+ protected static FileSystem createInternalFS(URI uri, Configuration conf)
+ throws IOException {
+ if (!schemeClassMap.containsKey(uri.getScheme())) {
+ throw new IOException("No FileSystem for scheme: " + uri.getScheme());
+ }
+ FileSystem fs = null;
+ try {
+ Class<? extends FileSystem> clazz =
+ (Class<? extends FileSystem>) conf.getClassByName(
+ schemeClassMap.get(uri.getScheme()));
+ fs = ReflectionUtils.newInstance(clazz, conf);
+ fs.initialize(uri, conf);
+ LOG.info("Create filesystem {} for scheme {} .",
+ schemeClassMap.get(uri.getScheme()), uri.getScheme());
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Can not found FileSystem Clazz for scheme: " + uri.getScheme());
+ }
+ return fs;
+ }
+
+ protected void createLocalCacheManager(URI name, Configuration conf) throws IOException{
+ mHadoopFileOpener = uriStatus -> this.fs.open(new Path(uriStatus.getPath()));
+ mAlluxioFileOpener = status -> new AlluxioHdfsInputStream(mHadoopFileOpener.open(status));
+
+ mAlluxioConf = HadoopUtils.toAlluxioConf(conf);
+ // Handle metrics
+ Properties metricsProperties = new Properties();
+ for (Map.Entry<String, String> entry : conf) {
+ metricsProperties.setProperty(entry.getKey(), entry.getValue());
+ }
+ MetricsSystem.startSinksFromConfig(new MetricsConfig(metricsProperties));
+ mCacheManager = CacheManager.Factory.get(mAlluxioConf);
+ if (mCacheManager == null) {
+ throw new IOException("CacheManager is null !");
+ }
+ }
+
+ @Override
+ public synchronized void initialize(URI name, Configuration conf) throws IOException {
+ this.originalScheme = name.getScheme();
+ // create internal FileSystem according to the scheme
+ this.fs = createInternalFS(name, conf);
+ this.statistics = (FileSystem.Statistics) ReflectionUtil.getFieldValue(this.fs,
+ "statistics");
+ if (null == this.statistics) {
+ LOG.info("======= original statistics is null.");
+ } else {
+ LOG.info("======= original statistics is {} {}.", this.statistics.getScheme(),
+ this.statistics.toString());
+ }
+ super.initialize(name, conf);
+ this.setConf(conf);
+ LOG.info("======= current statistics is {} {}.", this.statistics.getScheme(),
+ this.statistics.toString());
+
+ this.bufferSize = conf.getInt(CacheFileSystemConstants.PARAMS_KEY_IO_FILE_BUFFER_SIZE,
+ CacheFileSystemConstants.PARAMS_KEY_IO_FILE_BUFFER_SIZE_DEFAULT_VALUE);
+ // when scheme is jfs, use the cache by jfs itself
+ this.useLocalCache = conf.getBoolean(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE,
+ CacheFileSystemConstants.PARAMS_KEY_USE_CACHE_DEFAULT_VALUE)
+ && !this.originalScheme.equals(CacheFileSystemConstants.JUICEFS_SCHEME);
+
+ this.useLegacyFileInputStream = conf.getBoolean(
+ CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM,
+ CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM_DEFAULT_VALUE);
+
+ // create FileStatus cache
+ long fileStatusTTL =
+ conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL,
+ CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE);
+ long fileStatusMaxSize =
+ conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE,
+ CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE_DEFAULT_VALUE);
+ CacheLoader<Path, FileStatus> fileStatusCacheLoader = new CacheLoader<Path, FileStatus>() {
+ @Override
+ public FileStatus load(Path path) throws Exception {
+ return getFileStatusForCache(path);
+ }
+ };
+ this.fileStatusCache =
+ CacheBuilder.newBuilder()
+ .maximumSize(fileStatusMaxSize)
+ .expireAfterAccess(fileStatusTTL, TimeUnit.SECONDS)
+ .recordStats()
+ .build(fileStatusCacheLoader);
+
+ // create LocalCacheFileSystem if needs
+ if (this.isUseLocalCache()) {
+ // Todo: Can set local cache dir here for the current executor
+ this.createLocalCacheManager(this.getUri(), conf);
+ LOG.info("Create LocalCacheFileSystem successfully .");
+ }
+ }
+
+ protected FileStatus getFileStatusForCache(Path path) throws IOException {
+ return this.fs.getFileStatus(path);
+ }
+
+ @Override
+ public String getScheme() {
+ return this.originalScheme;
+ }
+
+ @Override
+ public FSDataInputStream open(Path f) throws IOException {
+ return open(f, bufferSize);
+ }
+
+ /**
+ * Check whether needs to cache data on the current executor
+ */
+ public abstract boolean isUseLocalCacheForTargetExecs();
+
+ /**
+ * Wrap FileStatus to Alluxio FileInfo
+ */
+ public FileInfo wrapFileInfo(FileStatus fileStatus) {
+ return (new FileInfo()
+ .setLength(fileStatus.getLen())
+ .setPath(fileStatus.getPath().toString())
+ .setFolder(fileStatus.isDirectory())
+ .setBlockSizeBytes(fileStatus.getBlockSize())
+ .setLastModificationTimeMs(fileStatus.getModificationTime())
+ .setLastAccessTimeMs(fileStatus.getAccessTime())
+ .setOwner(fileStatus.getOwner())
+ .setGroup(fileStatus.getGroup()));
+ }
+
+ private int checkBufferSize(int size) {
+ if (size < this.bufferSize) {
+ size = this.bufferSize;
+ }
+ // int numWords = (size + 7) / 8;
+ // int alignedSize = numWords * 8;
+ // assert (alignedSize >= size);
+ // return alignedSize;
+ return size;
+ }
+
+ @Override
+ public FSDataInputStream open(Path p, int bufferSize) throws IOException {
+ return this.open(p, bufferSize, this.isUseLocalCacheForTargetExecs());
+ }
+
+ public FSDataInputStream open(Path p, int bufferSize, boolean useLocalCacheForExec) throws IOException {
+ Path f = this.fs.makeQualified(p);
+
+ if (this.isUseLocalCache() && this.mCacheManager != null && useLocalCacheForExec) {
+ FileStatus fileStatus = this.getFileStatus(f);
+ FileInfo fileInfo = wrapFileInfo(fileStatus);
+ // FilePath is a unique identifier for a file, however it can be a long string
+ // hence using md5 hash of the file path as the identifier in the cache.
+ CacheContext context = CacheContext.defaults().setCacheIdentifier(
+ md5().hashString(fileStatus.getPath().toString(), UTF_8).toString());
+ URIStatus status = new URIStatus(fileInfo, context);
+ LOG.info("Use local cache FileSystem to open file {} .", f);
+ if (this.useLegacyFileInputStream) {
+ return new FSDataInputStream(new AlluxioHdfsFileInputStream(
+ new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager,
+ mAlluxioConf), statistics));
+ }
+ return new FSDataInputStream(new CacheFileInputStream(f,
+ new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager,
+ mAlluxioConf),
+ null, statistics, checkBufferSize(bufferSize)));
+ }
+ LOG.info("Use original FileSystem to open file {} .", f);
+ return super.open(f, bufferSize);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ this.statistics.incrementReadOps(1);
+ long start = System.currentTimeMillis();
+ FileStatus fileStatus = null;
+ Path p = this.fs.makeQualified(f);
+ try {
+ fileStatus = this.fileStatusCache.get(p);
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof FileNotFoundException)
+ throw new FileNotFoundException("File does not exist: " + p);
+ LOG.error("Get file status from cache error: " + p, e);
+ return fileStatus;
+ }
+ LOG.info("Get file status {} from cache took: {}", f,
+ (System.currentTimeMillis() - start));
+ return fileStatus;
+ }
+
+ public CacheManager getmCacheManager() {
+ return mCacheManager;
+ }
+
+ public void setmCacheManager(CacheManager mCacheManager) {
+ this.mCacheManager = mCacheManager;
+ }
+
+ public AlluxioConfiguration getmAlluxioConf() {
+ return mAlluxioConf;
+ }
+
+ public void setmAlluxioConf(AlluxioConfiguration mAlluxioConf) {
+ this.mAlluxioConf = mAlluxioConf;
+ }
+
+ public boolean isUseLocalCache() {
+ return useLocalCache;
+ }
+
+ public void setUseLocalCache(boolean useLocalCache) {
+ this.useLocalCache = useLocalCache;
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public void setBufferSize(int bufferSize) {
+ this.bufferSize = bufferSize;
+ }
+
+ public boolean isUseLegacyFileInputStream() {
+ return useLegacyFileInputStream;
+ }
+
+ public void setUseLegacyFileInputStream(boolean useLegacyFileInputStream) {
+ this.useLegacyFileInputStream = useLegacyFileInputStream;
+ }
+
+ public LoadingCache<Path, FileStatus> getFileStatusCache() {
+ return fileStatusCache;
+ }
+}
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
new file mode 100644
index 0000000000..f94a5dffe1
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import alluxio.AlluxioURI;
+import alluxio.client.file.FileInStream;
+import alluxio.client.file.FileSystem;
+import alluxio.exception.AlluxioException;
+import alluxio.exception.ExceptionMessage;
+import alluxio.exception.FileDoesNotExistException;
+
+import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+@NotThreadSafe
+public class AlluxioHdfsFileInputStream extends InputStream implements Seekable, PositionedReadable,
+ ByteBufferReadable {
+ private static final Logger LOG = LoggerFactory.getLogger(AlluxioHdfsFileInputStream.class);
+
+ private final Statistics mStatistics;
+ private final FileInStream mInputStream;
+
+ private boolean mClosed = false;
+
+ /**
+ * Constructs a new stream for reading a file from HDFS.
+ *
+ * @param fs the file system
+ * @param uri the Alluxio file URI
+ * @param stats filesystem statistics
+ */
+ public AlluxioHdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats)
+ throws IOException {
+ LOG.debug("HdfsFileInputStream({}, {})", uri, stats);
+
+ mStatistics = stats;
+ try {
+ mInputStream = fs.openFile(uri);
+ } catch (FileDoesNotExistException e) {
+ // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract.
+ throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri));
+ } catch (AlluxioException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Constructs a new stream for reading a file from HDFS.
+ *
+ * @param inputStream the input stream
+ * @param stats filesystem statistics
+ */
+ public AlluxioHdfsFileInputStream(FileInStream inputStream, Statistics stats) {
+ mInputStream = inputStream;
+ mStatistics = stats;
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (mClosed) {
+ throw new IOException("Cannot query available bytes from a closed stream.");
+ }
+ return (int) mInputStream.remaining();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (mClosed) {
+ return;
+ }
+ mInputStream.close();
+ mClosed = true;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return mInputStream.getPos();
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (mClosed) {
+ throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
+ }
+
+ int data = mInputStream.read();
+ if (data != -1 && mStatistics != null) {
+ mStatistics.incrementBytesRead(1);
+ LOG.info("Read one byte.");
+ }
+ return data;
+ }
+
+ @Override
+ public int read(byte[] buffer) throws IOException {
+ return read(buffer, 0, buffer.length);
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ if (mClosed) {
+ throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
+ }
+
+ int bytesRead = mInputStream.read(buffer, offset, length);
+ if (bytesRead != -1 && mStatistics != null) {
+ mStatistics.incrementBytesRead(bytesRead);
+ LOG.info("Read {} bytes.", bytesRead);
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public int read(ByteBuffer buf) throws IOException {
+ if (mClosed) {
+ throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
+ }
+ int bytesRead;
+ if (buf.hasArray() || !buf.isDirect()) {
+ bytesRead = mInputStream.read(buf.array(), buf.position(), buf.remaining());
+ if (bytesRead > 0) {
+ buf.position(buf.position() + bytesRead);
+ }
+ } else {
+ bytesRead = mInputStream.read(buf);
+ }
+ if (bytesRead != -1 && mStatistics != null) {
+ mStatistics.incrementBytesRead(bytesRead);
+ LOG.info("Read {} byte buffer {}.", bytesRead, buf.hasArray());
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ if (mClosed) {
+ throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
+ }
+
+ int bytesRead = mInputStream.positionedRead(position, buffer, offset, length);
+ if (bytesRead != -1 && mStatistics != null) {
+ mStatistics.incrementBytesRead(bytesRead);
+ LOG.info("Read {} {} byte buffer.", position, bytesRead);
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ readFully(position, buffer, 0, buffer.length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+ int totalBytesRead = 0;
+ while (totalBytesRead < length) {
+ int bytesRead =
+ read(position + totalBytesRead, buffer, offset + totalBytesRead, length - totalBytesRead);
+ if (bytesRead == -1) {
+ throw new EOFException();
+ }
+ totalBytesRead += bytesRead;
+ }
+ LOG.info("Read fully {} {} byte buffer.", position, totalBytesRead);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ try {
+ mInputStream.seek(pos);
+ } catch (IllegalArgumentException e) { // convert back to IOException
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * This method is not supported in {@link HdfsFileInputStream}.
+ *
+ * @param targetPos N/A
+ * @return N/A
+ * @throws IOException always
+ */
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ throw new IOException(ExceptionMessage.NOT_SUPPORTED.getMessage());
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (mClosed) {
+ throw new IOException("Cannot skip bytes in a closed stream.");
+ }
+ return mInputStream.skip(n);
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileInputStream.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileInputStream.java
new file mode 100644
index 0000000000..185e7364a9
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileInputStream.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import alluxio.client.file.FileInStream;
+import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.DirectBufferPool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class CacheFileInputStream extends FSInputStream implements ByteBufferReadable {
+
+ public static final int EINVAL = -0x16;
+ /**
+ * Refer to the code of 'FileInputStream' from JuiceFS Java SDK
+ */
+
+ private static final Logger LOG = LoggerFactory.getLogger(CacheFileInputStream.class);
+
+ private final DirectBufferPool bufferPool;
+ private final Statistics statistics;
+ private final FileInStream mInputStream;
+
+ private ByteBuffer buf;
+ private Path file;
+ private boolean mClosed = false;
+
+ public CacheFileInputStream(Path file, FileInStream inputStream,
+ DirectBufferPool directBufferPool,
+ Statistics statistics, int size) throws IOException {
+ this.file = file;
+ this.mInputStream = inputStream;
+ this.bufferPool = directBufferPool;
+ if (this.bufferPool != null) {
+ this.buf = this.bufferPool.getBuffer(size);
+ } else {
+ this.buf = ByteBuffer.allocate(size);
+ }
+ this.statistics = statistics;
+ this.buf.limit(0);
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ return mInputStream.getPos() - buf.remaining();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ return buf.remaining() + mInputStream.available();
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ throw new IOException("Mark/reset not supported");
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining() && !refill())
+ return -1; // EOF
+ assert buf.hasRemaining();
+ statistics.incrementBytesRead(1);
+ return buf.get() & 0xFF;
+ }
+
+ @Override
+ public synchronized int read(byte[] b, int off, int len) throws IOException {
+ if (off < 0 || len < 0 || b.length - off < len)
+ throw new IndexOutOfBoundsException();
+ if (len == 0)
+ return 0;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining() && len <= buf.capacity() && !refill())
+ return -1; // No bytes were read before EOF.
+
+ int read = Math.min(buf.remaining(), len);
+ if (read > 0) {
+ buf.get(b, off, read);
+ statistics.incrementBytesRead(read);
+ off += read;
+ len -= read;
+ }
+ if (len == 0)
+ return read;
+ // buf is empty, read data from mInputStream directly
+ int more = readInternal(b, off, len);
+ if (more <= 0) {
+ if (read > 0) {
+ return read;
+ } else {
+ return -1;
+ }
+ }
+ // read byte[], buf must be empty
+ buf.position(0);
+ buf.limit(0);
+ return read + more;
+ }
+
+ protected synchronized int readInternal(byte[] b, int off, int len) throws IOException {
+ if (len == 0)
+ return 0;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (b == null || off < 0 || len < 0 || b.length - off < len) {
+ throw new IllegalArgumentException(
+ "Reading file " + this.file.toString() + " error, invalid arguments: " +
+ off + " " + len);
+ }
+ int got = mInputStream.read(b, off, len);
+ if (got == 0)
+ return -1;
+ if (got == EINVAL)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (got < 0)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ statistics.incrementBytesRead(got);
+ return got;
+ }
+
+ private boolean refill() throws IOException {
+ buf.clear();
+ int read = readInternal(buf);
+ if (read <= 0) {
+ buf.limit(0);
+ return false; // EOF
+ }
+ statistics.incrementBytesRead(-read);
+ buf.flip();
+ return true;
+ }
+
+ @Override
+ public synchronized int read(long pos, byte[] b, int off, int len) throws IOException {
+ if (len == 0)
+ return 0;
+ if (pos < 0)
+ throw new EOFException(
+ "Reading file " + this.file.toString() + " error, position is negative");
+ if (b == null || off < 0 || len < 0 || b.length - off < len) {
+ throw new IllegalArgumentException(
+ "Reading file " + this.file.toString() + " error, invalid arguments: " +
+ off + " " + len);
+ }
+ long oldPos = mInputStream.getPos();
+ mInputStream.seek(pos);
+ int got = -1;
+ try {
+ got = mInputStream.read(b, off, len);
+ if (got == 0)
+ return -1;
+ if (got == EINVAL)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (got < 0)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ } finally {
+ mInputStream.seek(oldPos);
+ }
+ statistics.incrementBytesRead(got);
+ return got;
+ }
+
+ @Override
+ public synchronized int read(ByteBuffer b) throws IOException {
+ if (!b.hasRemaining())
+ return 0;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining() && b.remaining() <= buf.capacity() && !refill()) {
+ return -1;
+ }
+ int got = Math.min(b.remaining(), buf.remaining());
+ if (got > 0) {
+ byte[] readedBytes = new byte[got];
+ buf.get(readedBytes, 0, got);
+ b.put(readedBytes, 0, got);
+ statistics.incrementBytesRead(got);
+ }
+ if (!b.hasRemaining())
+ return got;
+ int more = readInternal(b);
+ if (more <= 0)
+ return got > 0 ? got : -1;
+ buf.position(0);
+ buf.limit(0);
+ return got + more;
+ }
+
+ protected synchronized int readInternal(ByteBuffer b) throws IOException {
+ if (!b.hasRemaining())
+ return 0;
+ int got;
+ if (b.hasArray()) {
+ // for heap bytebuffer
+ got = readInternal(b.array(), b.position(), b.remaining());
+ if (got <= 0)
+ return got;
+ b.position(b.position() + got);
+ } else {
+ assert b.isDirect();
+ got = mInputStream.read(b, b.position(), b.remaining());
+ if (got == EINVAL)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (got < 0)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (got == 0)
+ return -1;
+ statistics.incrementBytesRead(got);
+ }
+ return got;
+ }
+
+ @Override
+ public synchronized void seek(long p) throws IOException {
+ if (p < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (p < mInputStream.getPos() && p >= mInputStream.getPos() - buf.limit()) {
+ buf.position((int) (p - (mInputStream.getPos() - buf.limit())));
+ } else {
+ buf.position(0);
+ buf.limit(0);
+ mInputStream.seek(p);
+ }
+ }
+
+ @Override
+ public synchronized long skip(long n) throws IOException {
+ if (n < 0)
+ return -1;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (n < buf.remaining()) {
+ buf.position(buf.position() + (int) n);
+ } else {
+ mInputStream.skip(n - buf.remaining());
+ buf.position(0);
+ buf.limit(0);
+ }
+ return n;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (!mClosed) {
+ mInputStream.close();
+ mClosed = true;
+ }
+ if (this.bufferPool != null && buf != null) {
+ this.bufferPool.returnBuffer(buf);
+ }
+ buf = null;
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
new file mode 100644
index 0000000000..45985a3f7c
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+public class CacheFileSystemConstants {
+
+ private CacheFileSystemConstants() {
+ }
+
+ public static final String PARAMS_KEY_USE_CACHE =
+ "spark.kylin.local-cache.enabled";
+
+ public static final boolean PARAMS_KEY_USE_CACHE_DEFAULT_VALUE = false;
+
+ public static final String PARAMS_KEY_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ public static final int PARAMS_KEY_IO_FILE_BUFFER_SIZE_DEFAULT_VALUE = 65536;
+
+ public static final String PARAMS_KEY_FILE_STATUS_CACHE_TTL =
+ "spark.kylin.local-cache.filestatus.cache.ttl";
+
+ public static final long PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE = 3600L;
+
+ public static final String PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE =
+ "spark.kylin.local-cache.filestatus.cache.max-size";
+
+ public static final long PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE_DEFAULT_VALUE = 10000L;
+
+ public static final String PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM =
+ "spark.kylin.local-cache.use.legacy.file.input-stream";
+
+ public static final boolean PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM_DEFAULT_VALUE = false;
+
+ public static final String PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES =
+ "spark.kylin.local-cache.for.current.files";
+
+ public static final String JUICEFS_SCHEME = "jfs";
+}
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/MemCacheFileInputStream.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/MemCacheFileInputStream.java
new file mode 100644
index 0000000000..aba546d04e
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/MemCacheFileInputStream.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class MemCacheFileInputStream extends FSInputStream implements ByteBufferReadable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemCacheFileInputStream.class);
+
+ private final Statistics statistics;
+ private ByteBuffer buf;
+ private Path file;
+ private int fileLength;
+
+ public MemCacheFileInputStream(Path file, ByteBuffer buf, int fileLength,
+ Statistics statistics) throws IOException {
+ this.file = file;
+ this.buf = buf;
+ this.fileLength = fileLength;
+ this.statistics = statistics;
+ assert this.fileLength == buf.capacity();
+
+ // reset ByteBuffer
+ this.buf.flip();
+ this.buf.limit(fileLength);
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ return buf.position();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ return buf.remaining();
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ throw new IOException("Mark/reset not supported");
+ }
+
+ public synchronized int read() throws IOException {
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining())
+ return -1; // EOF
+ statistics.incrementBytesRead(1);
+ return buf.get() & 0xFF;
+ }
+
+ @Override
+ public synchronized int read(byte[] b, int off, int len) throws IOException {
+ if (off < 0 || len < 0 || b.length - off < len)
+ throw new IndexOutOfBoundsException();
+ if (len == 0)
+ return 0;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining())
+ return -1; // No bytes were read before EOF.
+
+ int read = Math.min(buf.remaining(), len);
+ if (read > 0) {
+ buf.get(b, off, read);
+ statistics.incrementBytesRead(read);
+ }
+ return read;
+ }
+
+ @Override
+ public synchronized int read(long pos, byte[] b, int off, int len) throws IOException {
+ if (len == 0)
+ return 0;
+ if (pos < 0 || pos > buf.limit())
+ throw new EOFException(
+ "Reading file " + this.file.toString() + " error, position is negative");
+ if (b == null || off < 0 || len < 0 || b.length - off < len) {
+ throw new IllegalArgumentException(
+ "Reading file " + this.file.toString() + " error, invalid arguments: " +
+ off + " " + len);
+ }
+ int oldPos = buf.position();
+ buf.position((int)pos);
+ int got = Math.min(buf.remaining(), len);
+ try {
+ if (got > 0) {
+ buf.get(b, off, len);
+ statistics.incrementBytesRead(got);
+ }
+ } finally {
+ buf.position(oldPos);
+ }
+ return got;
+ }
+
+ @Override
+ public synchronized int read(ByteBuffer b) throws IOException {
+ if (!b.hasRemaining())
+ return 0;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (!buf.hasRemaining()) {
+ return -1;
+ }
+ int got = Math.min(b.remaining(), buf.remaining());
+ if (got > 0) {
+ byte[] readedBytes = new byte[got];
+ buf.get(readedBytes, 0, got);
+ b.put(readedBytes, 0, got);
+ statistics.incrementBytesRead(got);
+ }
+ return got;
+ }
+
+ @Override
+ public synchronized void seek(long p) throws IOException {
+ if (p < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ if (p > buf.limit()) {
+ throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+ }
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ buf.position((int)p);
+ }
+
+ @Override
+ public synchronized long skip(long n) throws IOException {
+ if (n < 0)
+ return -1;
+ if (buf == null)
+ throw new IOException(
+ "Reading file " + this.file.toString() + " error, stream was closed");
+ if (n > buf.remaining()) {
+ throw new EOFException("Attempted to skip past the end of the file");
+ }
+ buf.position(buf.position() + (int) n);
+ return n;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (buf != null) {
+ buf = null;
+ }
+ }
+
+ public Statistics getStatistics() {
+ return statistics;
+ }
+
+ public ByteBuffer getBuf() {
+ return buf;
+ }
+
+ public void setBuf(ByteBuffer buf) {
+ this.buf = buf;
+ }
+
+ public Path getFile() {
+ return file;
+ }
+
+ public void setFile(Path file) {
+ this.file = file;
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/OnlyForTestCacheFileSystem.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/OnlyForTestCacheFileSystem.java
new file mode 100644
index 0000000000..8017266559
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/OnlyForTestCacheFileSystem.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+public class OnlyForTestCacheFileSystem extends AbstractCacheFileSystem {
+
+ /**
+ * Check whether needs to cache data on the current executor
+ */
+ @Override
+ public boolean isUseLocalCacheForTargetExecs() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/CacheAllFileSystem.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/CacheAllFileSystem.java
new file mode 100644
index 0000000000..a5af07ac32
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/CacheAllFileSystem.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs.kylin;
+
+import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+
+public class CacheAllFileSystem extends AbstractCacheFileSystem {
+
+ /**
+ * Check whether needs to cache data on the current executor
+ */
+ @Override
+ public boolean isUseLocalCacheForTargetExecs() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/KylinCacheFileSystem.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/KylinCacheFileSystem.java
new file mode 100644
index 0000000000..e47bff9b18
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/kylin/KylinCacheFileSystem.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs.kylin;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+import org.apache.kylin.cache.fs.CacheFileSystemConstants;
+import org.apache.spark.TaskContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KylinCacheFileSystem extends AbstractCacheFileSystem {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KylinCacheFileSystem.class);
+
+ /**
+ * Check whether needs to cache data on the current executor
+ */
+ @Override
+ public boolean isUseLocalCacheForTargetExecs() {
+ if (null == TaskContext.get()) {
+ LOG.warn("Task Context is null.");
+ return false;
+ }
+ String localCacheForCurrExecutor =
+ TaskContext.get()
+ .getLocalProperty(CacheFileSystemConstants.PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES);
+ LOG.info("Cache for current executor is {}", localCacheForCurrExecutor);
+ return (StringUtils.isNotBlank(localCacheForCurrExecutor) && Boolean.valueOf(localCacheForCurrExecutor));
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ConsistentHash.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ConsistentHash.java
new file mode 100644
index 0000000000..7f3af8bae7
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ConsistentHash.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.utils;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+import java.util.List;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+public class ConsistentHash<T> {
+
+ private final int numberOfVirtualNodeReplicas;
+ private final SortedMap<Integer, T> circle = new ConcurrentSkipListMap<>();
+ private final HashFunction nodeHash = Hashing.murmur3_32();
+ private final HashFunction keyHash = Hashing.murmur3_32();
+
+ public ConsistentHash(int numberOfVirtualNodeReplicas, List<T> nodes) {
+ this.numberOfVirtualNodeReplicas = numberOfVirtualNodeReplicas;
+ addNode(nodes);
+ }
+
+ public ConsistentHash(int numberOfVirtualNodeReplicas) {
+ this.numberOfVirtualNodeReplicas = numberOfVirtualNodeReplicas;
+ }
+
+ public void addNode(List<T> nodes) {
+ for (T node : nodes) {
+ addNode(node);
+ }
+ }
+
+ public void addNode(T node) {
+ for (int i = 0; i < numberOfVirtualNodeReplicas; i++) {
+ circle.put(getKetamaHash(i + "" + node), node);
+ }
+ }
+
+ public void remove(List<T> nodes) {
+ for (T node : nodes) {
+ remove(node);
+ }
+ }
+
+ public void remove(T node) {
+ for (int i = 0; i < numberOfVirtualNodeReplicas; i++) {
+ circle.remove(getKetamaHash(i + "" + node));
+ }
+ }
+
+ public T get(Object key) {
+ if (circle.isEmpty()) {
+ return null;
+ }
+ int hash = getKeyHash(key.toString());
+ if (!circle.containsKey(hash)) {
+ SortedMap<Integer, T> tailMap = circle.tailMap(hash);
+ hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey();
+ }
+ return circle.get(hash);
+ }
+
+ private int getKeyHash(final String k) {
+ return keyHash.hashBytes(k.getBytes()).asInt();
+ }
+
+ private int getKetamaHash(final String k) {
+ return nodeHash.hashBytes(k.getBytes()).asInt();
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
new file mode 100644
index 0000000000..d2f5fba234
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+
+public class ReflectionUtil {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ReflectionUtil.class);
+
+ private ReflectionUtil() {
+ }
+
+ /**
+ * Get the value of the specified field
+ *
+ */
+ public static Object getFieldValue(Object object, String fieldName) {
+ Field field = getDeclaredField(object, fieldName);
+ if (field == null) {
+ throw new IllegalArgumentException(
+ "Could not find field [" + fieldName + "] on target [" + object + "]");
+ }
+ makeAccessible(field);
+
+ Object result = null;
+ try {
+ result = field.get(object);
+ } catch (IllegalAccessException e) {
+ LOG.error("Get field value error:", e);
+ }
+
+ return result;
+ }
+
+ /**
+ * Set the value of the specified field
+ *
+ */
+ public static void setFieldValue(Object object, String fieldName, Object value) {
+ Field field = getDeclaredField(object, fieldName);
+ if (field == null) {
+ throw new IllegalArgumentException(
+ "Could not find field [" + fieldName + "] on target [" + object + "]");
+ }
+ makeAccessible(field);
+
+ try {
+ field.set(object, value);
+ } catch (IllegalAccessException e) {
+ LOG.error("Set field value error:", e);
+ }
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("unchecked")
+ public static Class getSuperClassGenricType(Class clazz, int index) {
+ Type genType = clazz.getGenericSuperclass();
+ if (!(genType instanceof ParameterizedType)) {
+ return Object.class;
+ }
+
+ Type[] params = ((ParameterizedType) genType).getActualTypeArguments();
+ if (index >= params.length || index < 0) {
+ return Object.class;
+ }
+ if (!(params[index] instanceof Class)) {
+ return Object.class;
+ }
+
+ return (Class) params[index];
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> Class<T> getSuperGenericType(Class clazz) {
+ return getSuperClassGenricType(clazz, 0);
+ }
+
+ /**
+ *
+ */
+ public static Method getDeclaredMethod(Object object, String methodName, Class<?>[] parameterTypes) {
+ for (Class<?> superClass = object.getClass(); superClass != Object.class;
+ superClass = superClass.getSuperclass()) {
+ try {
+ return superClass.getDeclaredMethod(methodName, parameterTypes);
+ } catch (NoSuchMethodException e) {
+ }
+ }
+ LOG.error("Can not find method " + methodName);
+ return null;
+ }
+
+ /**
+ *
+ */
+ public static void makeAccessible(Field field) {
+ if (!Modifier.isPublic(field.getModifiers())) {
+ field.setAccessible(true);
+ }
+ }
+
+ /**
+ *
+ */
+ public static Field getDeclaredField(Object object, String filedName) {
+ for (Class<?> superClass = object.getClass(); superClass != Object.class;
+ superClass = superClass.getSuperclass()) {
+ try {
+ return superClass.getDeclaredField(filedName);
+ } catch (NoSuchFieldException e) {
+ }
+ }
+ LOG.error("Can not find field " + filedName);
+ return null;
+ }
+
+ /**
+ * Invoke method
+ *
+ */
+ public static Object invokeMethod(Object object, String methodName, Class<?>[] parameterTypes,
+ Object[] parameters) throws InvocationTargetException {
+ Method method = getDeclaredMethod(object, methodName, parameterTypes);
+ if (method == null) {
+ throw new IllegalArgumentException(
+ "Could not find method [" + methodName + "] on target [" + object + "]");
+ }
+ method.setAccessible(true);
+
+ try {
+ return method.invoke(object, parameters);
+ } catch (IllegalAccessException e) {
+ LOG.error("Invoke method " + methodName + " error: ", e);
+ }
+
+ return null;
+ }
+}
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/softaffinity/SoftAffinityConstants.java b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/softaffinity/SoftAffinityConstants.java
new file mode 100644
index 0000000000..2f9e968d15
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/softaffinity/SoftAffinityConstants.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.softaffinity;
+
+public class SoftAffinityConstants {
+
+ private SoftAffinityConstants() {
+ }
+
+ public static final String PARAMS_KEY_SOFT_AFFINITY_ENABLED =
+ "spark.kylin.soft-affinity.enabled";
+
+ public static final boolean PARAMS_KEY_SOFT_AFFINITY_ENABLED_DEFAULT_VALUE = false;
+
+ public static final String PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM =
+ "spark.kylin.soft-affinity.replications.num";
+
+ public static final int PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM_DEFAULT_VALUE = 2;
+
+ // For HDFS Replications
+ public static final String PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS =
+ "spark.kylin.soft-affinity.min.target-hosts";
+
+ public static final int PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS_DEFAULT_VALUE = 1;
+}
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
new file mode 100644
index 0000000000..720f5576e4
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.softaffinity
+
+import org.apache.kylin.softaffinity.strategy.SoftAffinityStrategy
+
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import scala.collection.mutable
+import org.apache.spark.SparkEnv
+import org.apache.spark.internal.Logging
+
+import java.util.concurrent.atomic.AtomicInteger
+
+object SoftAffinityManager extends Logging {
+
+ val resourceRWLock = new ReentrantReadWriteLock(true)
+
+ val softAffinityAllocation = new SoftAffinityStrategy
+
+ lazy val minOnTargetHosts = SparkEnv.get.conf.getInt(
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS,
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS_DEFAULT_VALUE
+ )
+
+ // (execId, host) list
+ val fixedIdForExecutors = new mutable.ListBuffer[Option[(String, String)]]()
+ // host list
+ val nodesExecutorsMap = new mutable.HashMap[String, mutable.HashSet[String]]()
+
+ protected val totalRegisteredExecutors = new AtomicInteger(0)
+
+ lazy val usingSoftAffinity = SparkEnv.get.conf.getBoolean(
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED,
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED_DEFAULT_VALUE
+ )
+
+ def totalExecutors(): Int = totalRegisteredExecutors.intValue()
+
+ def handleExecutorAdded(execHostId: (String, String)): Unit = {
+ resourceRWLock.writeLock().lock()
+ try {
+ // first, check whether the execId exists
+ if (!fixedIdForExecutors.exists( exec => {
+ exec.isDefined && exec.get._1.equals(execHostId._1)
+ })) {
+ val executorsSet = nodesExecutorsMap.getOrElseUpdate(execHostId._2,
+ new mutable.HashSet[String]())
+ executorsSet.add(execHostId._1)
+ if (fixedIdForExecutors.exists(_.isEmpty)) {
+ // replace the executor which was removed
+ val replaceIdx = fixedIdForExecutors.indexWhere(_.isEmpty)
+ fixedIdForExecutors(replaceIdx) = Option(execHostId)
+ } else {
+ fixedIdForExecutors += Option(execHostId)
+ }
+ totalRegisteredExecutors.addAndGet(1)
+ }
+ logInfo(s"After adding executor ${execHostId._1} on host ${execHostId._2}, " +
+ s"fixedIdForExecutors is ${fixedIdForExecutors.mkString(",")}, " +
+ s"nodesExecutorsMap is ${nodesExecutorsMap.keySet.mkString(",")}, " +
+ s"actual executors count is ${totalRegisteredExecutors.intValue()}."
+ )
+ } finally {
+ resourceRWLock.writeLock().unlock()
+ }
+ }
+
+ def handleExecutorRemoved(execId: String): Unit = {
+ resourceRWLock.writeLock().lock()
+ try {
+ val execIdx = fixedIdForExecutors.indexWhere( execHost => {
+ if (execHost.isDefined) {
+ execHost.get._1.equals(execId)
+ } else {
+ false
+ }
+ })
+ if (execIdx != -1) {
+ val findedExecId = fixedIdForExecutors(execIdx)
+ fixedIdForExecutors(execIdx) = None
+ val nodeExecs = nodesExecutorsMap.get(findedExecId.get._2).get
+ nodeExecs -= findedExecId.get._1
+ if (nodeExecs.isEmpty) {
+ // there is no executor on this host, remove
+ nodesExecutorsMap.remove(findedExecId.get._2)
+ }
+ totalRegisteredExecutors.addAndGet(-1)
+ }
+ logInfo(s"After removing executor ${execId}, " +
+ s"fixedIdForExecutors is ${fixedIdForExecutors.mkString(",")}, " +
+ s"nodesExecutorsMap is ${nodesExecutorsMap.keySet.mkString(",")}, " +
+ s"actual executors count is ${totalRegisteredExecutors.intValue()}."
+ )
+ } finally {
+ resourceRWLock.writeLock().unlock()
+ }
+ }
+
+ def checkTargetHosts(hosts: Array[String]): Boolean = {
+ resourceRWLock.readLock().lock()
+ try {
+ if (hosts.length < 1) {
+ // there is no host locality
+ false
+ } else if (nodesExecutorsMap.size < 1) {
+ true
+ } else {
+ // when the replication num of hdfs is less than 'minOnTargetHosts'
+ val minHostsNum = Math.min(minOnTargetHosts, hosts.length)
+ // there are how many the same hosts
+ nodesExecutorsMap.map(_._1).toArray.intersect(hosts).size >= minHostsNum
+ }
+ } finally {
+ resourceRWLock.readLock().unlock()
+ }
+ }
+
+ def askExecutors(file: String): Array[(String, String)] = {
+ resourceRWLock.readLock().lock()
+ try {
+ if (nodesExecutorsMap.size < 1) {
+ Array.empty
+ } else {
+ softAffinityAllocation.allocateExecs(file, fixedIdForExecutors)
+ }
+ } finally {
+ resourceRWLock.readLock().unlock()
+ }
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/scheduler/SoftAffinityListener.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/scheduler/SoftAffinityListener.scala
new file mode 100644
index 0000000000..6d1b5ac1a1
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/scheduler/SoftAffinityListener.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.softaffinity.scheduler
+
+import org.apache.kylin.softaffinity.SoftAffinityManager
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorAdded, SparkListenerExecutorRemoved}
+
+class SoftAffinityListener extends SparkListener with Logging {
+
+ override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = {
+ val execId = executorAdded.executorId
+ val host = executorAdded.executorInfo.executorHost
+ SoftAffinityManager.handleExecutorAdded((execId, host))
+ }
+
+ override def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit = {
+ val execId = executorRemoved.executorId
+ SoftAffinityManager.handleExecutorRemoved(execId)
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityAllocationTrait.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityAllocationTrait.scala
new file mode 100644
index 0000000000..06f9d284fb
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityAllocationTrait.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.softaffinity.strategy
+
+import org.apache.kylin.softaffinity.SoftAffinityConstants
+
+import scala.collection.mutable.ListBuffer
+import org.apache.spark.SparkEnv
+
+trait SoftAffinityAllocationTrait {
+
+ lazy val softAffinityReplicationNum = SparkEnv.get.conf.getInt(
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM,
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM_DEFAULT_VALUE
+ )
+
+ /**
+ * allocate target executors for file
+ */
+ def allocateExecs(file: String,
+ candidates: ListBuffer[Option[(String, String)]]): Array[(String, String)]
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityStrategy.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityStrategy.scala
new file mode 100644
index 0000000000..a01641245d
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/strategy/SoftAffinityStrategy.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.softaffinity.strategy
+
+import scala.collection.mutable.LinkedHashSet
+import scala.collection.mutable.ListBuffer
+
+import org.apache.spark.internal.Logging
+
+class SoftAffinityStrategy extends SoftAffinityAllocationTrait with Logging {
+
+ /**
+ * allocate target executors for file
+ */
+ override def allocateExecs(file: String,
+ candidates: ListBuffer[Option[(String, String)]]): Array[(String, String)] = {
+ if (candidates.size < 1) {
+ Array.empty
+ } else {
+ val candidatesSize = candidates.size
+ val halfCandidatesSize = candidatesSize / softAffinityReplicationNum
+ val resultSet = new LinkedHashSet[(String, String)]
+
+ val mod = file.hashCode % candidatesSize
+ val c1 = if (mod < 0) (mod + candidatesSize) else mod
+ // check whether the executor with index c1 is down
+ if (candidates(c1).isDefined) {
+ resultSet.add(candidates(c1).get)
+ }
+ for (i <- 1 to (softAffinityReplicationNum - 1)) {
+ val c2 = (c1 + halfCandidatesSize + i) % candidatesSize
+ if (candidates(c2).isDefined) {
+ resultSet.add(candidates(c2).get)
+ }
+ }
+ resultSet.toArray
+ }
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
new file mode 100644
index 0000000000..38a0dfa76c
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import org.apache.kylin.softaffinity.SoftAffinityManager
+
+import org.apache.spark.Partition
+import org.apache.spark.internal.Logging
+import org.apache.spark.scheduler.ExecutorCacheTaskLocation
+import org.apache.spark.sql.connector.read.InputPartition
+
+/**
+ * A collection of file blocks that should be read as a single task
+ * (possibly from multiple partitioned directories).
+ */
+case class CacheFilePartition(index: Int, files: Array[CachePartitionedFile])
+ extends Partition with InputPartition {
+ override def preferredLocations(): Array[String] = {
+ // Todo: Only get the first one
+ files.head.locations.map { p =>
+ if (p._1.equals("")) p._2
+ else ExecutorCacheTaskLocation(p._2, p._1).toString
+ }.toArray
+ }
+}
+
+object CacheFilePartition extends Logging {
+
+ def convertFilePartitionToCache(filePartition: FilePartition): CacheFilePartition = {
+ // Get the original preferred locations
+ val expectedTargets = filePartition.preferredLocations()
+ val files = filePartition.files
+
+ var locations = Array.empty[(String, String)]
+ if (!files.isEmpty && SoftAffinityManager.usingSoftAffinity
+ && !SoftAffinityManager.checkTargetHosts(expectedTargets)) {
+ // if there is no host in the node list which are executors running on,
+ // using SoftAffinityManager to generate target executors.
+ // Only using the first file to calculate the target executors
+ locations = SoftAffinityManager.askExecutors(files.head.filePath)
+ if (!locations.isEmpty) {
+ logInfo(s"SAMetrics=File ${files.head.filePath} - " +
+ s"the expected executors are ${locations.mkString("_")} ")
+ } else {
+ locations = expectedTargets.map(("", _))
+ }
+ } else {
+ locations = expectedTargets.map(("", _))
+ }
+ CacheFilePartition(filePartition.index, filePartition.files.map(p => {
+ CachePartitionedFile(p.partitionValues, p.filePath, p.start, p.length, locations)
+ }))
+ }
+}
+
+
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFileScanRDD.scala b/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFileScanRDD.scala
new file mode 100644
index 0000000000..75f7ed4abc
--- /dev/null
+++ b/kylin-spark-project/kylin-soft-affinity-cache/src/main/spark31/org/apache/spark/sql/execution/datasources/CacheFileScanRDD.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import org.apache.kylin.cache.fs.CacheFileSystemConstants
+
+import org.apache.spark.{SparkEnv, TaskContext, Partition => RDDPartition}
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.InternalRow
+
+case class CachePartitionedFile(
+ partitionValues: InternalRow,
+ filePath: String,
+ start: Long,
+ length: Long,
+ locations: Array[(String, String)] = Array.empty) {
+ override def toString: String = {
+ s"path: $filePath, range: $start-${start + length}, partition values: $partitionValues," +
+ s" on executor locations ${locations.mkString}"
+ }
+}
+
+class CacheFileScanRDD(
+ @transient private val sparkSession: SparkSession,
+ readFunction: (PartitionedFile) => Iterator[InternalRow],
+ @transient val cacheFilePartitions: Seq[CacheFilePartition])
+ extends FileScanRDD(sparkSession, readFunction, Nil) {
+
+ def checkCached(cacheLocations: Array[(String, String)]): Boolean = {
+ cacheLocations.map(_._1).contains(SparkEnv.get.executorId)
+ }
+
+ override def compute(split: RDDPartition, context: TaskContext): Iterator[InternalRow] = {
+ // Convert 'CacheFilePartition' to 'FilePartition'
+ val start = System.currentTimeMillis()
+ val cacheFilePartition = split.asInstanceOf[CacheFilePartition]
+ val cacheSplit = FilePartition(cacheFilePartition.index, cacheFilePartition.files.map { f =>
+ PartitionedFile(f.partitionValues, f.filePath, f.start, f.length, f.locations.map(_._2))
+ })
+ var currFilePath = "empty"
+ val isCache = if (!cacheFilePartition.files.isEmpty) {
+ currFilePath = cacheFilePartition.files.head.filePath
+ checkCached(cacheFilePartition.files.head.locations)
+ } else {
+ false
+ }
+ logInfo(s"SAMetrics=File ${currFilePath} running in task ${context.taskAttemptId()} " +
+ s"on executor ${SparkEnv.get.executorId} with cached ${isCache} , " +
+ s"took ${(System.currentTimeMillis() - start)}")
+ // Set whether needs to cache data on this executor
+ context.getLocalProperties.setProperty(
+ CacheFileSystemConstants.PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES, isCache.toString)
+ super.compute(cacheSplit, context)
+ }
+
+ override protected def getPartitions: Array[RDDPartition] = cacheFilePartitions.toArray
+
+ override protected def getPreferredLocations(split: RDDPartition): Seq[String] = {
+ split.asInstanceOf[CacheFilePartition].preferredLocations()
+ }
+}
diff --git a/kylin-spark-project/kylin-spark-common/pom.xml b/kylin-spark-project/kylin-spark-common/pom.xml
index 4d0cee3851..0e62aa27e6 100644
--- a/kylin-spark-project/kylin-spark-common/pom.xml
+++ b/kylin-spark-project/kylin-spark-common/pom.xml
@@ -48,6 +48,11 @@
<artifactId>kylin-spark-metadata</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-soft-affinity-cache</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
diff --git a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/AbstractHdfsLogAppender.java b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/AbstractHdfsLogAppender.java
index 17422a841d..b0b2301596 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/AbstractHdfsLogAppender.java
+++ b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/AbstractHdfsLogAppender.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;
+import org.apache.spark.utils.SparkHadoopUtils;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -91,7 +92,7 @@ public abstract class AbstractHdfsLogAppender extends AppenderSkeleton {
public FileSystem getFileSystem() {
if (null == fileSystem) {
- return getFileSystem(new Configuration());
+ return getFileSystem(SparkHadoopUtils.newConfigurationWithSparkConf());
}
return fileSystem;
}
diff --git a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkDriverHdfsLogAppender.java b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkDriverHdfsLogAppender.java
index 0e5ad3c318..94a44092d0 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkDriverHdfsLogAppender.java
+++ b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkDriverHdfsLogAppender.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;
+import org.apache.spark.utils.SparkHadoopUtils;
import java.io.IOException;
import java.util.List;
@@ -92,7 +93,7 @@ public class SparkDriverHdfsLogAppender extends AbstractHdfsLogAppender {
public void doWriteLog(int eventSize, List<LoggingEvent> transaction)
throws IOException, InterruptedException {
if (!isWriterInited()) {
- Configuration conf = new Configuration();
+ Configuration conf = SparkHadoopUtils.newConfigurationWithSparkConf();
if (isKerberosEnable()) {
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(getKerberosPrincipal(), getKerberosKeytab());
diff --git a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkExecutorHdfsAppender.java b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkExecutorHdfsAppender.java
index f341340e61..d28ae16b21 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkExecutorHdfsAppender.java
+++ b/kylin-spark-project/kylin-spark-common/src/main/java/org/apache/kylin/engine/spark/common/logging/SparkExecutorHdfsAppender.java
@@ -20,7 +20,6 @@ package org.apache.kylin.engine.spark.common.logging;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -30,6 +29,7 @@ import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.spark.SparkEnv;
import org.apache.spark.deploy.SparkHadoopUtil;
+import org.apache.spark.utils.SparkHadoopUtils;
import scala.runtime.BoxedUnit;
import java.io.File;
@@ -160,7 +160,7 @@ public class SparkExecutorHdfsAppender extends AbstractHdfsLogAppender {
SparkHadoopUtil.get().runAsSparkUser(new scala.runtime.AbstractFunction0<scala.runtime.BoxedUnit>() {
@Override
public BoxedUnit apply() {
- if (!initHdfsWriter(file, new Configuration())) {
+ if (!initHdfsWriter(file, SparkHadoopUtils.newConfigurationWithSparkConf())) {
LogLog.error("Failed to init the hdfs writer!");
}
try {
diff --git a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/utils/SparkHadoopUtils.scala b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/utils/SparkHadoopUtils.scala
new file mode 100644
index 0000000000..0594b1c05a
--- /dev/null
+++ b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/utils/SparkHadoopUtils.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.utils
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.SparkEnv
+import org.apache.spark.deploy.SparkHadoopUtil
+
+object SparkHadoopUtils {
+
+ def newConfigurationWithSparkConf(): Configuration = {
+ SparkHadoopUtil.newConfiguration(SparkEnv.get.conf)
+ }
+}
\ No newline at end of file
diff --git a/kylin-spark-project/kylin-spark-common/src/main/spark31/org/apache/spark/sql/execution/KylinFileSourceScanExec.scala b/kylin-spark-project/kylin-spark-common/src/main/spark31/org/apache/spark/sql/execution/KylinFileSourceScanExec.scala
index b924f3afd8..61fef995fe 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/spark31/org/apache/spark/sql/execution/KylinFileSourceScanExec.scala
+++ b/kylin-spark-project/kylin-spark-common/src/main/spark31/org/apache/spark/sql/execution/KylinFileSourceScanExec.scala
@@ -21,6 +21,7 @@ package org.apache.spark.sql.execution
import org.apache.hadoop.fs.{BlockLocation, FileStatus, LocatedFileStatus, Path}
import org.apache.kylin.common.KylinConfig
import org.apache.spark.rdd.RDD
+import org.apache.kylin.softaffinity.SoftAffinityManager
import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, Expression, ExpressionUtils, SortOrder}
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning}
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
@@ -165,7 +166,14 @@ class KylinFileSourceScanExec(
FilePartition(shardId, filesToPartitionId.getOrElse(shardId, Nil).toArray)
}
- new FileScanRDD(fsRelation.sparkSession, readFile, filePartitions)
+ if (SoftAffinityManager.usingSoftAffinity) {
+ val start = System.currentTimeMillis()
+ val cachePartitions = filePartitions.map(CacheFilePartition.convertFilePartitionToCache(_))
+ logInfo(s"Convert bucketed file partition took: ${(System.currentTimeMillis() - start)}")
+ new CacheFileScanRDD(fsRelation.sparkSession, readFile, cachePartitions)
+ } else {
+ new FileScanRDD(fsRelation.sparkSession, readFile, filePartitions)
+ }
}
/**
@@ -237,7 +245,14 @@ class KylinFileSourceScanExec(
}
closePartition()
- new FileScanRDD(fsRelation.sparkSession, readFile, partitions)
+ if (SoftAffinityManager.usingSoftAffinity) {
+ val start = System.currentTimeMillis()
+ val cachePartitions = partitions.map(CacheFilePartition.convertFilePartitionToCache(_))
+ logInfo(s"Convert file partition took: ${(System.currentTimeMillis() - start)}")
+ new CacheFileScanRDD(fsRelation.sparkSession, readFile, cachePartitions)
+ } else {
+ new FileScanRDD(fsRelation.sparkSession, readFile, partitions)
+ }
}
private def getBlockLocations(file: FileStatus): Array[BlockLocation] = file match {
diff --git a/kylin-spark-project/pom.xml b/kylin-spark-project/pom.xml
index 2031d2c5ee..5d2c241872 100644
--- a/kylin-spark-project/pom.xml
+++ b/kylin-spark-project/pom.xml
@@ -37,6 +37,7 @@
<module>kylin-spark-query</module>
<module>kylin-spark-test</module>
<module>kylin-spark-classloader</module>
+ <module>kylin-soft-affinity-cache</module>
</modules>
<dependencies>
[kylin] 03/03: Upgrade alluxio client verion to 2.7.4
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch kylin-soft-affinity-local-cache
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 65bd536741eb1cfd2c55c8f7e5dcfe87c3bdbdc8
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Sat Apr 16 10:40:30 2022 +0800
Upgrade alluxio client verion to 2.7.4
---
kylin-spark-project/kylin-soft-affinity-cache/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/kylin-spark-project/kylin-soft-affinity-cache/pom.xml b/kylin-spark-project/kylin-soft-affinity-cache/pom.xml
index 87370db8d4..03df75b6e3 100644
--- a/kylin-spark-project/kylin-soft-affinity-cache/pom.xml
+++ b/kylin-spark-project/kylin-soft-affinity-cache/pom.xml
@@ -27,13 +27,13 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-spark-project</artifactId>
- <version>4.0.0-SNAPSHOT</version>
+ <version>4.0.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <alluxio.version>2.6.1</alluxio.version>
+ <alluxio.version>2.7.4</alluxio.version>
</properties>
<dependencies>
[kylin] 02/03: Pre-init KylinCacheFileSystem to fix s3a issue
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch kylin-soft-affinity-local-cache
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b893faf8bda8a17712777e539d4da187403486e7
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Sat Sep 25 23:44:38 2021 +0800
Pre-init KylinCacheFileSystem to fix s3a issue
---
.../org/apache/kylin/common/KylinConfigBase.java | 7 ++++-
.../sql/execution/datasource/FilePruner.scala | 9 +++++++
.../org/apache/spark/sql/SparderContext.scala | 31 +++++++++++-----------
3 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 31a600857f..ab242ff974 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -37,6 +37,7 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.text.StrSubstitutor;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.annotation.ConfigTag;
@@ -295,6 +296,10 @@ public abstract class KylinConfigBase implements Serializable {
}
public String getHdfsWorkingDirectory() {
+ return getHdfsWorkingDirectoryInternal(HadoopUtil.getCurrentConfiguration());
+ }
+
+ public String getHdfsWorkingDirectoryInternal(Configuration hadoopConf) {
if (cachedHdfsWorkingDirectory != null) {
return cachedHdfsWorkingDirectory;
}
@@ -306,7 +311,7 @@ public abstract class KylinConfigBase implements Serializable {
throw new IllegalArgumentException("kylin.env.hdfs-working-dir must be absolute, but got " + root);
try {
- FileSystem fs = path.getFileSystem(HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = path.getFileSystem(hadoopConf);
path = fs.makeQualified(path);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
index 0c5cfbff7b..34fc967c29 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
+++ b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
@@ -211,15 +211,20 @@ class FilePruner(cubeInstance: CubeInstance,
var cached = new java.util.HashMap[(Seq[Expression], Seq[Expression]), Seq[PartitionDirectory]]()
private def getFileStatusBySeg(seg: SegmentDirectory, fsc: FileStatusCache): SegmentDirectory = {
+ var startT = System.currentTimeMillis()
val path = new Path(toPath(seg.segmentName, seg.identifier))
val fs = path.getFileSystem(session.sparkContext.hadoopConfiguration)
+ logInfo(s"Get segment filesystem: ${System.currentTimeMillis() - startT}")
+ startT = System.currentTimeMillis()
if (fs.isDirectory(path) && fs.exists(path)) {
val maybeStatuses = fsc.getLeafFiles(path)
if (maybeStatuses.isDefined) {
+ logInfo(s"Get segment status from cache: ${System.currentTimeMillis() - startT}")
SegmentDirectory(seg.segmentName, seg.identifier, maybeStatuses.get)
} else {
val statuses = fs.listStatus(path)
fsc.putLeafFiles(path, statuses)
+ logInfo(s"Get segment status and cache: ${System.currentTimeMillis() - startT}")
SegmentDirectory(seg.segmentName, seg.identifier, statuses)
}
} else {
@@ -239,17 +244,21 @@ class FilePruner(cubeInstance: CubeInstance,
val timePartitionFilters = getSegmentFilter(dataFilters, timePartitionColumn)
logInfo(s"Applying time partition filters: ${timePartitionFilters.mkString(",")}")
+ var startT = System.currentTimeMillis()
val fsc = ShardFileStatusCache.getFileStatusCache(session)
+ logInfo(s"Get file status cache: ${System.currentTimeMillis() - startT}")
// segment pruning
var selected = afterPruning("segment", timePartitionFilters, segmentDirs) {
pruneSegments
}
+ startT = System.currentTimeMillis()
// fetch segment directories info in parallel
selected = selected.par.map(seg => {
getFileStatusBySeg(seg, fsc)
}).filter(_.files.nonEmpty).seq
+ logInfo(s"Get segment status: ${System.currentTimeMillis() - startT}")
// shards pruning
selected = afterPruning("shard", dataFilters, selected) {
diff --git a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
index faa921230d..e6b73d66cc 100644
--- a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
+++ b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
@@ -18,30 +18,28 @@
package org.apache.spark.sql
-import java.lang.{Boolean => JBoolean, String => JString}
-import java.nio.file.Paths
-
-import org.apache.spark.memory.MonitorEnv
-import org.apache.spark.util.Utils
-import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
-import org.apache.kylin.query.UdfManager
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.parser.ParseException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.KylinSession._
-import java.util.concurrent.atomic.AtomicReference
-
import org.apache.commons.io.FileUtils
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.util.ToolUtil
+import org.apache.kylin.query.UdfManager
import org.apache.kylin.query.monitor.SparderContextCanary
import org.apache.kylin.spark.classloader.ClassLoaderUtils
import org.apache.spark.deploy.StandaloneAppClient
-import org.apache.spark.sql.SparderContext.master_app_url
-import org.apache.spark.{SparkConf, SparkContext, SparkEnv}
+import org.apache.spark.internal.Logging
+import org.apache.spark.memory.MonitorEnv
+import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
+import org.apache.spark.sql.KylinSession._
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasource.{KylinSourceStrategy, ShardFileStatusCache}
import org.apache.spark.sql.metrics.SparderMetricsListener
+import org.apache.spark.util.Utils
import org.apache.spark.utils.YarnInfoFetcherUtils
+import org.apache.spark.{SparkConf, SparkContext, SparkEnv}
+
+import java.lang.{Boolean => JBoolean, String => JString}
+import java.nio.file.Paths
+import java.util.concurrent.atomic.AtomicReference
// scalastyle:off
object SparderContext extends Logging {
@@ -192,6 +190,9 @@ object SparderContext extends Logging {
case _ =>
master_app_url = YarnInfoFetcherUtils.getTrackingUrl(appid)
}
+
+ // pre-init FileSystem, fix s3a issue
+ kylinConf.getHdfsWorkingDirectoryInternal(spark.sparkContext.hadoopConfiguration)
} catch {
case throwable: Throwable =>
logError("Error for initializing spark ", throwable)