You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ph...@apache.org on 2024/04/01 05:20:33 UTC
(incubator-gluten) branch main updated: [CORE] Change domain name to org.apache.gluten (#5185)
This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cf252bcd3 [CORE] Change domain name to org.apache.gluten (#5185)
cf252bcd3 is described below
commit cf252bcd368432722ff1aa2ffbfa6f3890a3694f
Author: Yan Ma <ya...@intel.com>
AuthorDate: Mon Apr 1 13:20:28 2024 +0800
[CORE] Change domain name to org.apache.gluten (#5185)
Change gluten domain name to org.apache.gluten. User needs to note the plugin configuration is affected.
--conf spark.plugins=org.apache.gluten.GlutenPlugin
---
.github/workflows/velox_be.yml.deprecated | 12 +-
.github/workflows/velox_docker.yml | 12 +-
.scalafmt.conf | 2 +-
README.md | 4 +-
backends-clickhouse/pom.xml | 6 +-
.../commands/OptimizeTableCommandOverwrites.scala | 2 +-
.../commands/OptimizeTableCommandOverwrites.scala | 2 +-
.../execution/ColumnarNativeIterator.java | 73 -
.../glutenproject/execution/SparkRowIterator.java | 83 -
.../alloc/CHManagedCHReservationListener.java | 152 --
.../memory/alloc/CHNativeMemoryAllocator.java | 71 -
.../alloc/CHNativeMemoryAllocatorManager.java | 24 -
.../alloc/CHNativeMemoryAllocatorManagerImpl.java | 46 -
.../memory/alloc/CHNativeMemoryAllocators.java | 101 -
.../memory/alloc/CHReservationListener.java | 53 -
.../java/io/glutenproject/metrics/MetricsData.java | 138 --
.../io/glutenproject/metrics/MetricsProcessor.java | 105 -
.../java/io/glutenproject/metrics/MetricsStep.java | 85 -
.../io/glutenproject/metrics/NativeMetrics.java | 73 -
.../io/glutenproject/metrics/OperatorMetrics.java | 37 -
.../io/glutenproject/utils/SnowflakeIdWorker.java | 109 -
.../io/glutenproject/vectorized/BatchIterator.java | 79 -
.../vectorized/BlockOutputStream.java | 96 -
.../vectorized/BlockSplitIterator.java | 135 --
.../vectorized/CHBlockConverterJniWrapper.java | 40 -
.../vectorized/CHBlockWriterJniWrapper.java | 51 -
.../glutenproject/vectorized/CHColumnVector.java | 146 --
.../io/glutenproject/vectorized/CHNativeBlock.java | 104 -
.../vectorized/CHNativeExpressionEvaluator.java | 120 -
.../vectorized/CHShuffleSplitterJniWrapper.java | 124 -
.../glutenproject/vectorized/CHStreamReader.java | 66 -
.../vectorized/ExpressionEvaluatorJniWrapper.java | 60 -
.../glutenproject/vectorized/IteratorWrapper.java | 35 -
.../LowCopyFileSegmentShuffleInputStream.java | 99 -
.../vectorized/LowCopyNettyShuffleInputStream.java | 74 -
.../vectorized/OnHeapCopyShuffleInputStream.java | 79 -
.../vectorized/ShuffleInputStream.java | 35 -
.../vectorized/SimpleExpressionEval.java | 65 -
.../vectorized/StorageJoinBuilder.java | 100 -
.../gluten/execution/ColumnarNativeIterator.java | 73 +
.../apache/gluten/execution/SparkRowIterator.java | 83 +
.../alloc/CHManagedCHReservationListener.java | 152 ++
.../memory/alloc/CHNativeMemoryAllocator.java | 71 +
.../alloc/CHNativeMemoryAllocatorManager.java | 24 +
.../alloc/CHNativeMemoryAllocatorManagerImpl.java | 46 +
.../memory/alloc/CHNativeMemoryAllocators.java | 101 +
.../gluten/memory/alloc/CHReservationListener.java | 53 +
.../org/apache/gluten/metrics/MetricsData.java | 138 ++
.../apache/gluten/metrics/MetricsProcessor.java | 105 +
.../org/apache/gluten/metrics/MetricsStep.java | 85 +
.../org/apache/gluten/metrics/NativeMetrics.java | 73 +
.../org/apache/gluten/metrics/OperatorMetrics.java | 37 +
.../org/apache/gluten/utils/SnowflakeIdWorker.java | 109 +
.../apache/gluten/vectorized/BatchIterator.java | 79 +
.../gluten/vectorized/BlockOutputStream.java | 96 +
.../gluten/vectorized/BlockSplitIterator.java | 135 ++
.../vectorized/CHBlockConverterJniWrapper.java | 40 +
.../gluten/vectorized/CHBlockWriterJniWrapper.java | 51 +
.../apache/gluten/vectorized/CHColumnVector.java | 146 ++
.../apache/gluten/vectorized/CHNativeBlock.java | 104 +
.../vectorized/CHNativeExpressionEvaluator.java | 120 +
.../vectorized/CHShuffleSplitterJniWrapper.java | 124 +
.../apache/gluten/vectorized/CHStreamReader.java | 66 +
.../vectorized/ExpressionEvaluatorJniWrapper.java | 60 +
.../apache/gluten/vectorized/IteratorWrapper.java | 35 +
.../LowCopyFileSegmentShuffleInputStream.java | 99 +
.../vectorized/LowCopyNettyShuffleInputStream.java | 74 +
.../vectorized/OnHeapCopyShuffleInputStream.java | 79 +
.../gluten/vectorized/ShuffleInputStream.java | 35 +
.../gluten/vectorized/SimpleExpressionEval.java | 65 +
.../gluten/vectorized/StorageJoinBuilder.java | 100 +
.../spark/storage/CHShuffleReadStreamFactory.java | 10 +-
.../spark/storage/CHShuffleWriteStreamFactory.java | 2 +-
.../services/io.glutenproject.backendsapi.Backend | 1 -
.../services/org.apache.gluten.backendsapi.Backend | 1 +
.../backendsapi/clickhouse/CHBackend.scala | 287 ---
.../backendsapi/clickhouse/CHBroadcastApi.scala | 45 -
.../backendsapi/clickhouse/CHIteratorApi.scala | 366 ---
.../backendsapi/clickhouse/CHListenerApi.scala | 84 -
.../backendsapi/clickhouse/CHMetricsApi.scala | 372 ---
.../clickhouse/CHSparkPlanExecApi.scala | 816 -------
.../backendsapi/clickhouse/CHTransformerApi.scala | 245 --
.../backendsapi/clickhouse/CHValidatorApi.scala | 138 --
.../execution/CHBroadcastBuildSideCache.scala | 98 -
.../execution/CHFilterExecTransformer.scala | 51 -
.../execution/CHGenerateExecTransformer.scala | 91 -
.../execution/CHHashAggregateExecTransformer.scala | 473 ----
.../execution/CHHashJoinExecTransformer.scala | 125 -
.../execution/GlutenMergeTreePartition.scala | 54 -
.../execution/NativeFileScanColumnarRDD.scala | 106 -
.../expression/CHExpressionTransformer.scala | 224 --
.../extension/FallbackBroadcaseHashJoinRules.scala | 210 --
.../metrics/BatchScanMetricsUpdater.scala | 69 -
.../metrics/ExpandMetricsUpdater.scala | 51 -
.../metrics/FileSourceScanMetricsUpdater.scala | 83 -
.../metrics/FilterMetricsUpdater.scala | 51 -
.../metrics/GenerateMetricsUpdater.scala | 51 -
.../metrics/HashAggregateMetricsUpdater.scala | 86 -
.../metrics/HashJoinMetricsUpdater.scala | 117 -
.../metrics/HiveTableScanMetricsUpdater.scala | 64 -
.../metrics/InputIteratorMetricsUpdater.scala | 51 -
.../metrics/LimitMetricsUpdater.scala | 35 -
.../io/glutenproject/metrics/MetricsUtil.scala | 210 --
.../metrics/ProjectMetricsUpdater.scala | 51 -
.../metrics/SortMergeJoinMetricsUpdater.scala | 26 -
.../glutenproject/metrics/SortMetricsUpdater.scala | 52 -
.../metrics/WindowMetricsUpdater.scala | 51 -
.../io/glutenproject/utils/CHExpressionUtil.scala | 187 --
.../utils/CHInputPartitionsUtil.scala | 179 --
.../glutenproject/utils/CHJoinValidateUtil.scala | 104 -
.../io/glutenproject/utils/ExpressionDocUtil.scala | 142 --
.../io/glutenproject/utils/IteratorUtil.scala | 39 -
.../io/glutenproject/utils/PlanNodesUtil.scala | 76 -
.../vectorized/CHColumnarBatchSerializer.scala | 187 --
.../io/glutenproject/vectorized/CHSplitResult.java | 59 -
.../CloseableCHColumnBatchIterator.scala | 73 -
.../CloseablePartitionedBlockIterator.scala | 57 -
.../gluten/backendsapi/clickhouse/CHBackend.scala | 287 +++
.../backendsapi/clickhouse/CHBroadcastApi.scala | 45 +
.../backendsapi/clickhouse/CHIteratorApi.scala | 366 +++
.../backendsapi/clickhouse/CHListenerApi.scala | 84 +
.../backendsapi/clickhouse/CHMetricsApi.scala | 372 +++
.../clickhouse/CHSparkPlanExecApi.scala | 816 +++++++
.../backendsapi/clickhouse/CHTransformerApi.scala | 245 ++
.../backendsapi/clickhouse/CHValidatorApi.scala | 138 ++
.../execution/CHBroadcastBuildSideCache.scala | 98 +
.../gluten/execution/CHFilterExecTransformer.scala | 51 +
.../execution/CHGenerateExecTransformer.scala | 91 +
.../execution/CHHashAggregateExecTransformer.scala | 473 ++++
.../execution/CHHashJoinExecTransformer.scala | 125 +
.../execution/GlutenMergeTreePartition.scala | 54 +
.../execution/NativeFileScanColumnarRDD.scala | 106 +
.../expression/CHExpressionTransformer.scala | 224 ++
.../extension/FallbackBroadcaseHashJoinRules.scala | 210 ++
.../gluten/metrics/BatchScanMetricsUpdater.scala | 69 +
.../gluten/metrics/ExpandMetricsUpdater.scala | 51 +
.../metrics/FileSourceScanMetricsUpdater.scala | 83 +
.../gluten/metrics/FilterMetricsUpdater.scala | 51 +
.../gluten/metrics/GenerateMetricsUpdater.scala | 51 +
.../metrics/HashAggregateMetricsUpdater.scala | 86 +
.../gluten/metrics/HashJoinMetricsUpdater.scala | 117 +
.../metrics/HiveTableScanMetricsUpdater.scala | 64 +
.../metrics/InputIteratorMetricsUpdater.scala | 51 +
.../gluten/metrics/LimitMetricsUpdater.scala | 35 +
.../org/apache/gluten/metrics/MetricsUtil.scala | 210 ++
.../gluten/metrics/ProjectMetricsUpdater.scala | 51 +
.../metrics/SortMergeJoinMetricsUpdater.scala | 26 +
.../apache/gluten/metrics/SortMetricsUpdater.scala | 52 +
.../gluten/metrics/WindowMetricsUpdater.scala | 51 +
.../org/apache/gluten/utils/CHExpressionUtil.scala | 187 ++
.../gluten/utils/CHInputPartitionsUtil.scala | 179 ++
.../apache/gluten/utils/CHJoinValidateUtil.scala | 104 +
.../apache/gluten/utils/ExpressionDocUtil.scala | 142 ++
.../org/apache/gluten/utils/IteratorUtil.scala | 39 +
.../org/apache/gluten/utils/PlanNodesUtil.scala | 76 +
.../vectorized/CHColumnarBatchSerializer.scala | 187 ++
.../apache/gluten/vectorized/CHSplitResult.java | 59 +
.../CloseableCHColumnBatchIterator.scala | 73 +
.../CloseablePartitionedBlockIterator.scala | 57 +
.../org/apache/spark/affinity/CHAffinity.scala | 6 +-
.../spark/shuffle/CHColumnarShuffleWriter.scala | 10 +-
.../utils/RangePartitionerBoundsGenerator.scala | 14 +-
.../catalyst/CHAggregateFunctionRewriteRule.scala | 2 +-
.../delta/ClickhouseOptimisticTransaction.scala | 2 +-
.../spark/sql/execution/CHColumnarToRowExec.scala | 8 +-
.../sql/execution/RowToCHNativeColumnarExec.scala | 8 +-
.../utils/MergeTreePartsPartitionsUtil.scala | 4 +-
.../execution/datasources/v1/CHBlockStripes.java | 6 +-
.../datasources/v1/CHFormatWriterInjects.scala | 4 +-
.../datasources/v1/CHMergeTreeWriterInjects.scala | 12 +-
.../datasources/v1/CHParquetWriterInjects.scala | 2 +-
.../v1/GlutenMergeTreeWriterInjects.scala | 2 +-
.../clickhouse/MergeTreeFileFormatDataWriter.scala | 2 +-
.../v1/clickhouse/MergeTreeOutputWriter.scala | 2 +-
.../v2/clickhouse/ClickHouseSparkCatalog.scala | 2 +-
.../joins/ClickHouseBuildSideRelation.scala | 6 +-
.../spark/sql/execution/utils/CHExecUtil.scala | 12 +-
.../spark/memory/TestTaskMemoryManagerSuite.java | 10 +-
.../test/scala/io/glutenproject/RunTPCHTest.scala | 153 --
.../benchmarks/GenTPCDSDecimalTableScripts.scala | 1358 -----------
.../benchmarks/GenTPCDSTableScripts.scala | 1879 ---------------
.../benchmarks/GenTPCHTableScripts.scala | 762 ------
.../GlutenClickHouseColumnarShuffleAQESuite.scala | 174 --
.../GlutenClickHouseDSV2ColumnarShuffleSuite.scala | 177 --
.../execution/GlutenClickHouseDSV2Suite.scala | 164 --
.../GlutenClickHouseDatetimeExpressionSuite.scala | 164 --
.../execution/GlutenClickHouseDecimalSuite.scala | 417 ----
.../GlutenClickHouseFileFormatSuite.scala | 1352 -----------
.../execution/GlutenClickHouseHiveTableSuite.scala | 1255 ----------
.../GlutenClickHouseMergeTreeOptimizeSuite.scala | 336 ---
...GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala | 526 -----
...ergeTreeWriteOnObjectStorageAbstractSuite.scala | 188 --
.../GlutenClickHouseMergeTreeWriteOnS3Suite.scala | 543 -----
.../GlutenClickHouseMergeTreeWriteSuite.scala | 1556 ------------
.../GlutenClickHouseNativeExceptionSuite.scala | 42 -
.../GlutenClickHouseNativeWriteTableSuite.scala | 931 --------
...ckHousePreferSpillColumnarShuffleAQESuite.scala | 169 --
.../execution/GlutenClickHouseS3SourceSuite.scala | 136 --
.../GlutenClickHouseSyntheticDataSuite.scala | 236 --
.../GlutenClickHouseTPCDSAbstractSuite.scala | 236 --
.../GlutenClickHouseTPCDSParquetAQESuite.scala | 200 --
...kHouseTPCDSParquetColumnarShuffleAQESuite.scala | 243 --
...lickHouseTPCDSParquetColumnarShuffleSuite.scala | 205 --
...nClickHouseTPCDSParquetGraceHashJoinSuite.scala | 252 --
.../GlutenClickHouseTPCDSParquetRFSuite.scala | 45 -
.../GlutenClickHouseTPCDSParquetSuite.scala | 402 ----
.../GlutenClickHouseTPCHAbstractSuite.scala | 624 -----
.../GlutenClickHouseTPCHBucketSuite.scala | 806 -------
...ckHouseTPCHColumnarShuffleParquetAQESuite.scala | 351 ---
...GlutenClickHouseTPCHNotNullSkipIndexSuite.scala | 271 ---
...lickHouseTPCHNullableColumnarShuffleSuite.scala | 198 --
...lutenClickHouseTPCHNullableSkipIndexSuite.scala | 277 ---
.../GlutenClickHouseTPCHNullableSuite.scala | 213 --
...enClickHouseTPCHParquetAQEConcurrentSuite.scala | 83 -
.../GlutenClickHouseTPCHParquetAQESuite.scala | 376 ---
.../GlutenClickHouseTPCHParquetBucketSuite.scala | 808 -------
.../GlutenClickHouseTPCHParquetRFSuite.scala | 74 -
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 2491 --------------------
.../execution/GlutenClickHouseTPCHSuite.scala | 485 ----
.../GlutenClickHouseTableAfterRestart.scala | 224 --
...lutenClickHouseWholeStageTransformerSuite.scala | 75 -
.../GlutenClickhouseCountDistinctSuite.scala | 118 -
.../execution/GlutenClickhouseFunctionSuite.scala | 182 --
.../execution/GlutenFunctionValidateSuite.scala | 663 ------
.../extension/CustomAggExpressionTransformer.scala | 85 -
.../extension/GlutenCustomAggExpressionSuite.scala | 107 -
.../GlutenClickHouseMergeTreeMetricsSuite.scala | 263 ---
.../metrics/GlutenClickHouseMetricsUTUtils.scala | 113 -
.../GlutenClickHouseTPCDSMetricsSuite.scala | 127 -
.../metrics/GlutenClickHouseTPCHMetricsSuite.scala | 382 ---
.../scala/io/glutenproject/s3/S3AuthSuite.scala | 338 ---
.../glutenproject/utils/UTSystemParameters.scala | 51 -
.../test/scala/org/apache/gluten/RunTPCHTest.scala | 153 ++
.../benchmarks/GenTPCDSDecimalTableScripts.scala | 1358 +++++++++++
.../gluten/benchmarks/GenTPCDSTableScripts.scala | 1879 +++++++++++++++
.../gluten/benchmarks/GenTPCHTableScripts.scala | 762 ++++++
.../GlutenClickHouseColumnarShuffleAQESuite.scala | 174 ++
.../GlutenClickHouseDSV2ColumnarShuffleSuite.scala | 177 ++
.../execution/GlutenClickHouseDSV2Suite.scala | 164 ++
.../GlutenClickHouseDatetimeExpressionSuite.scala | 164 ++
.../execution/GlutenClickHouseDecimalSuite.scala | 417 ++++
.../GlutenClickHouseFileFormatSuite.scala | 1352 +++++++++++
.../execution/GlutenClickHouseHiveTableSuite.scala | 1255 ++++++++++
.../GlutenClickHouseMergeTreeOptimizeSuite.scala | 336 +++
...GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala | 526 +++++
...ergeTreeWriteOnObjectStorageAbstractSuite.scala | 188 ++
.../GlutenClickHouseMergeTreeWriteOnS3Suite.scala | 543 +++++
.../GlutenClickHouseMergeTreeWriteSuite.scala | 1556 ++++++++++++
.../GlutenClickHouseNativeExceptionSuite.scala | 42 +
.../GlutenClickHouseNativeWriteTableSuite.scala | 931 ++++++++
...ckHousePreferSpillColumnarShuffleAQESuite.scala | 169 ++
.../execution/GlutenClickHouseS3SourceSuite.scala | 136 ++
.../GlutenClickHouseSyntheticDataSuite.scala | 236 ++
.../GlutenClickHouseTPCDSAbstractSuite.scala | 236 ++
.../GlutenClickHouseTPCDSParquetAQESuite.scala | 200 ++
...kHouseTPCDSParquetColumnarShuffleAQESuite.scala | 243 ++
...lickHouseTPCDSParquetColumnarShuffleSuite.scala | 205 ++
...nClickHouseTPCDSParquetGraceHashJoinSuite.scala | 252 ++
.../GlutenClickHouseTPCDSParquetRFSuite.scala | 45 +
.../GlutenClickHouseTPCDSParquetSuite.scala | 402 ++++
.../GlutenClickHouseTPCHAbstractSuite.scala | 624 +++++
.../GlutenClickHouseTPCHBucketSuite.scala | 806 +++++++
...ckHouseTPCHColumnarShuffleParquetAQESuite.scala | 351 +++
...GlutenClickHouseTPCHNotNullSkipIndexSuite.scala | 271 +++
...lickHouseTPCHNullableColumnarShuffleSuite.scala | 198 ++
...lutenClickHouseTPCHNullableSkipIndexSuite.scala | 277 +++
.../GlutenClickHouseTPCHNullableSuite.scala | 213 ++
...enClickHouseTPCHParquetAQEConcurrentSuite.scala | 83 +
.../GlutenClickHouseTPCHParquetAQESuite.scala | 376 +++
.../GlutenClickHouseTPCHParquetBucketSuite.scala | 808 +++++++
.../GlutenClickHouseTPCHParquetRFSuite.scala | 74 +
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 2491 ++++++++++++++++++++
.../execution/GlutenClickHouseTPCHSuite.scala | 485 ++++
.../GlutenClickHouseTableAfterRestart.scala | 224 ++
...lutenClickHouseWholeStageTransformerSuite.scala | 75 +
.../GlutenClickhouseCountDistinctSuite.scala | 118 +
.../execution/GlutenClickhouseFunctionSuite.scala | 182 ++
.../execution/GlutenFunctionValidateSuite.scala | 663 ++++++
.../extension/CustomAggExpressionTransformer.scala | 85 +
.../extension/GlutenCustomAggExpressionSuite.scala | 107 +
.../GlutenClickHouseMergeTreeMetricsSuite.scala | 263 +++
.../metrics/GlutenClickHouseMetricsUTUtils.scala | 113 +
.../GlutenClickHouseTPCDSMetricsSuite.scala | 127 +
.../metrics/GlutenClickHouseTPCHMetricsSuite.scala | 382 +++
.../scala/org/apache/gluten/s3/S3AuthSuite.scala | 338 +++
.../apache/gluten/utils/UTSystemParameters.scala | 51 +
.../apache/spark/affinity/MixedAffinitySuite.scala | 6 +-
.../sql/execution/RemoveTopColumnarToRow.scala | 2 +-
.../benchmarks/CHAggAndShuffleBenchmark.scala | 4 +-
.../benchmarks/CHHashBuildBenchmark.scala | 4 +-
.../benchmarks/CHParquetReadBenchmark.scala | 14 +-
.../execution/benchmarks/CHSqlBasedBenchmark.scala | 8 +-
.../benchmarks/CHStorageJoinBenchmark.scala | 4 +-
.../parquet/GlutenParquetColumnIndexSuite.scala | 4 +-
.../gluten/parquet/GlutenParquetFilterSuite.scala | 2 +-
.../spark/sql/gluten/test/GlutenSQLTestUtils.scala | 2 +-
backends-velox/pom.xml | 14 +-
.../java/io/glutenproject/fs/JniFilesystem.java | 71 -
.../java/io/glutenproject/fs/OnHeapFileSystem.java | 279 ---
.../java/io/glutenproject/udf/UdfJniWrapper.java | 24 -
.../io/glutenproject/utils/GlutenURLDecoder.java | 118 -
.../java/org/apache/gluten/fs/JniFilesystem.java | 71 +
.../org/apache/gluten/fs/OnHeapFileSystem.java | 279 +++
.../java/org/apache/gluten/udf/UdfJniWrapper.java | 24 +
.../org/apache/gluten/utils/GlutenURLDecoder.java | 118 +
.../services/io.glutenproject.backendsapi.Backend | 1 -
.../services/org.apache.gluten.backendsapi.Backend | 1 +
.../backendsapi/velox/BroadcastApiImpl.scala | 32 -
.../backendsapi/velox/IteratorApiImpl.scala | 249 --
.../backendsapi/velox/ListenerApiImpl.scala | 196 --
.../backendsapi/velox/MetricsApiImpl.scala | 537 -----
.../backendsapi/velox/SparkPlanExecApiImpl.scala | 688 ------
.../backendsapi/velox/TransformerApiImpl.scala | 97 -
.../backendsapi/velox/ValidatorApiImpl.scala | 83 -
.../backendsapi/velox/VeloxBackend.scala | 506 ----
.../execution/FilterExecTransformer.scala | 44 -
.../execution/GenerateExecTransformer.scala | 203 --
...tenBroadcastNestedLoopJoinExecTransformer.scala | 49 -
.../execution/HashAggregateExecTransformer.scala | 826 -------
.../execution/RowToVeloxColumnarExec.scala | 240 --
.../ShuffledHashJoinExecTransformer.scala | 203 --
.../execution/VeloxBroadcastBuildSideRDD.scala | 37 -
.../execution/VeloxColumnarToRowExec.scala | 193 --
.../expression/ExpressionTransformer.scala | 146 --
.../io/glutenproject/utils/DatasourceUtil.scala | 57 -
.../glutenproject/utils/SharedLibraryLoader.scala | 23 -
.../utils/SharedLibraryLoaderCentos7.scala | 45 -
.../utils/SharedLibraryLoaderCentos8.scala | 49 -
.../utils/SharedLibraryLoaderDebian11.scala | 52 -
.../utils/SharedLibraryLoaderDebian12.scala | 58 -
.../utils/SharedLibraryLoaderUbuntu2004.scala | 68 -
.../utils/SharedLibraryLoaderUbuntu2204.scala | 53 -
.../utils/VeloxIntermediateData.scala | 155 --
.../backendsapi/velox/BroadcastApiImpl.scala | 32 +
.../gluten/backendsapi/velox/IteratorApiImpl.scala | 249 ++
.../gluten/backendsapi/velox/ListenerApiImpl.scala | 196 ++
.../gluten/backendsapi/velox/MetricsApiImpl.scala | 537 +++++
.../backendsapi/velox/SparkPlanExecApiImpl.scala | 688 ++++++
.../backendsapi/velox/TransformerApiImpl.scala | 97 +
.../backendsapi/velox/ValidatorApiImpl.scala | 83 +
.../gluten/backendsapi/velox/VeloxBackend.scala | 506 ++++
.../gluten/execution/FilterExecTransformer.scala | 44 +
.../gluten/execution/GenerateExecTransformer.scala | 203 ++
...tenBroadcastNestedLoopJoinExecTransformer.scala | 49 +
.../execution/HashAggregateExecTransformer.scala | 826 +++++++
.../gluten/execution/RowToVeloxColumnarExec.scala | 240 ++
.../ShuffledHashJoinExecTransformer.scala | 203 ++
.../execution/VeloxBroadcastBuildSideRDD.scala | 37 +
.../gluten/execution/VeloxColumnarToRowExec.scala | 193 ++
.../gluten/expression/ExpressionTransformer.scala | 146 ++
.../org/apache/gluten/utils/DatasourceUtil.scala | 57 +
.../apache/gluten/utils/SharedLibraryLoader.scala | 23 +
.../gluten/utils/SharedLibraryLoaderCentos7.scala | 45 +
.../gluten/utils/SharedLibraryLoaderCentos8.scala | 49 +
.../gluten/utils/SharedLibraryLoaderDebian11.scala | 52 +
.../gluten/utils/SharedLibraryLoaderDebian12.scala | 58 +
.../utils/SharedLibraryLoaderUbuntu2004.scala | 68 +
.../utils/SharedLibraryLoaderUbuntu2204.scala | 53 +
.../gluten/utils/VeloxIntermediateData.scala | 155 ++
.../catalyst/AggregateFunctionRewriteRule.scala | 2 +-
.../sql/catalyst/FlushableHashAggregateRule.scala | 2 +-
.../spark/sql/execution/BroadcastUtils.scala | 10 +-
.../execution/ColumnarCachedBatchSerializer.scala | 18 +-
.../execution/VeloxColumnarWriteFilesExec.scala | 10 +-
.../datasources/VeloxColumnarBatchIterator.scala | 2 +-
.../execution/datasources/VeloxWriteQueue.scala | 6 +-
.../datasources/velox/VeloxBlockStripes.java | 4 +-
.../velox/VeloxFormatWriterInjects.scala | 14 +-
.../velox/VeloxParquetWriterInjects.scala | 2 +-
.../apache/spark/sql/expression/UDFResolver.scala | 14 +-
.../io/glutenproject/fs/OnHeapFileSystemTest.java | 58 -
.../java/io/glutenproject/tags/FuzzerTest.java | 29 -
.../java/io/glutenproject/tags/SkipTestTags.java | 26 -
.../test/java/io/glutenproject/tags/UDFTest.java | 29 -
.../org/apache/gluten/fs/OnHeapFileSystemTest.java | 58 +
.../java/org/apache/gluten/tags/FuzzerTest.java | 29 +
.../java/org/apache/gluten/tags/SkipTestTags.java | 26 +
.../test/java/org/apache/gluten/tags/UDFTest.java | 29 +
.../benchmarks/NativeBenchmarkPlanGenerator.scala | 157 --
.../benchmarks/ShuffleWriterFuzzerTest.scala | 141 --
.../io/glutenproject/execution/FallbackSuite.scala | 178 --
.../execution/FunctionsValidateTest.scala | 78 -
.../execution/ScalarFunctionsValidateSuite.scala | 666 ------
.../io/glutenproject/execution/TestOperator.scala | 1239 ----------
.../execution/VeloxAggregateFunctionsSuite.scala | 990 --------
.../execution/VeloxColumnarCacheSuite.scala | 205 --
.../execution/VeloxHashJoinSuite.scala | 108 -
.../execution/VeloxLiteralSuite.scala | 140 --
.../execution/VeloxMetricsSuite.scala | 152 --
.../VeloxOrcDataTypeValidationSuite.scala | 475 ----
.../VeloxParquetDataTypeValidationSuite.scala | 485 ----
.../glutenproject/execution/VeloxScanSuite.scala | 77 -
.../execution/VeloxStringFunctionsSuite.scala | 596 -----
.../glutenproject/execution/VeloxTPCDSSuite.scala | 130 -
.../glutenproject/execution/VeloxTPCHSuite.scala | 370 ---
.../VeloxWholeStageTransformerSuite.scala | 21 -
.../execution/VeloxWindowExpressionSuite.scala | 72 -
.../execution/WindowFunctionsValidateSuite.scala | 35 -
.../glutenproject/expression/VeloxUdfSuite.scala | 134 --
.../io/glutenproject/planner/VeloxCboSuite.scala | 126 -
.../benchmarks/NativeBenchmarkPlanGenerator.scala | 157 ++
.../benchmarks/ShuffleWriterFuzzerTest.scala | 141 ++
.../apache/gluten/execution/FallbackSuite.scala | 178 ++
.../gluten/execution/FunctionsValidateTest.scala | 78 +
.../execution/ScalarFunctionsValidateSuite.scala | 666 ++++++
.../org/apache/gluten/execution/TestOperator.scala | 1239 ++++++++++
.../execution/VeloxAggregateFunctionsSuite.scala | 990 ++++++++
.../gluten/execution/VeloxColumnarCacheSuite.scala | 205 ++
.../gluten/execution/VeloxHashJoinSuite.scala | 108 +
.../gluten/execution/VeloxLiteralSuite.scala | 140 ++
.../gluten/execution/VeloxMetricsSuite.scala | 152 ++
.../VeloxOrcDataTypeValidationSuite.scala | 475 ++++
.../VeloxParquetDataTypeValidationSuite.scala | 485 ++++
.../apache/gluten/execution/VeloxScanSuite.scala | 77 +
.../execution/VeloxStringFunctionsSuite.scala | 596 +++++
.../apache/gluten/execution/VeloxTPCDSSuite.scala | 130 +
.../apache/gluten/execution/VeloxTPCHSuite.scala | 370 +++
.../VeloxWholeStageTransformerSuite.scala | 21 +
.../execution/VeloxWindowExpressionSuite.scala | 72 +
.../execution/WindowFunctionsValidateSuite.scala | 35 +
.../apache/gluten/expression/VeloxUdfSuite.scala | 134 ++
.../org/apache/gluten/planner/VeloxCboSuite.scala | 126 +
.../execution/VeloxParquetWriteForHiveSuite.scala | 4 +-
.../sql/execution/VeloxParquetWriteSuite.scala | 6 +-
.../benchmark/ColumnarTableCacheBenchmark.scala | 2 +-
.../benchmark/StreamingAggregateBenchmark.scala | 2 +-
cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp | 2 +-
cpp-ch/local-engine/jni/jni_error.cpp | 2 +-
cpp-ch/local-engine/libch-hide-jemalloc.map | 2 +-
cpp-ch/local-engine/libch.map | 2 +-
cpp-ch/local-engine/local_engine_jni.cpp | 160 +-
cpp/core/CMakeLists.txt | 2 +-
cpp/core/jni/JniCommon.cc | 2 +-
cpp/core/jni/JniCommon.h | 2 +-
cpp/core/jni/JniWrapper.cc | 122 +-
cpp/velox/jni/JniFileSystem.cc | 15 +-
cpp/velox/jni/VeloxJniWrapper.cc | 8 +-
cpp/velox/symbols.map | 4 +-
docs/Configuration.md | 4 +-
docs/developers/HowTo.md | 2 +-
docs/developers/MicroBenchmarks.md | 2 +-
docs/developers/NewToGluten.md | 2 +-
docs/get-started/ClickHouse.md | 8 +-
docs/get-started/Velox.md | 2 +-
.../src/resources/conf/gluten.properties | 2 +-
gluten-cbo/common/pom.xml | 2 +-
.../src/main/scala/io/glutenproject/cbo/Cbo.scala | 239 --
.../scala/io/glutenproject/cbo/CboCluster.scala | 84 -
.../scala/io/glutenproject/cbo/CboConfig.scala | 31 -
.../scala/io/glutenproject/cbo/CboExplain.scala | 21 -
.../main/scala/io/glutenproject/cbo/CboGroup.scala | 56 -
.../main/scala/io/glutenproject/cbo/CboNode.scala | 136 --
.../scala/io/glutenproject/cbo/CboPlanner.scala | 154 --
.../scala/io/glutenproject/cbo/CostModel.scala | 25 -
.../scala/io/glutenproject/cbo/MetadataModel.scala | 28 -
.../scala/io/glutenproject/cbo/PlanModel.scala | 32 -
.../scala/io/glutenproject/cbo/PropertyModel.scala | 42 -
.../io/glutenproject/cbo/best/BestFinder.scala | 68 -
.../cbo/best/GroupBasedBestFinder.scala | 121 -
.../io/glutenproject/cbo/dp/DpClusterAlgo.scala | 105 -
.../io/glutenproject/cbo/dp/DpGroupAlgo.scala | 95 -
.../scala/io/glutenproject/cbo/dp/DpPlanner.scala | 200 --
.../io/glutenproject/cbo/dp/DpZipperAlgo.scala | 656 ------
.../cbo/exaustive/ExhaustivePlanner.scala | 141 --
.../glutenproject/cbo/memo/ForwardMemoTable.scala | 203 --
.../scala/io/glutenproject/cbo/memo/Memo.scala | 227 --
.../io/glutenproject/cbo/memo/MemoTable.scala | 91 -
.../scala/io/glutenproject/cbo/path/CboPath.scala | 195 --
.../io/glutenproject/cbo/path/OutputFilter.scala | 116 -
.../io/glutenproject/cbo/path/OutputWizard.scala | 429 ----
.../io/glutenproject/cbo/path/PathFinder.scala | 191 --
.../scala/io/glutenproject/cbo/path/PathKey.scala | 60 -
.../scala/io/glutenproject/cbo/path/PathMask.scala | 192 --
.../scala/io/glutenproject/cbo/path/Pattern.scala | 133 --
.../glutenproject/cbo/property/PropertySet.scala | 68 -
.../scala/io/glutenproject/cbo/rule/CboRule.scala | 39 -
.../io/glutenproject/cbo/rule/EnforcerRule.scala | 93 -
.../io/glutenproject/cbo/rule/RuleApplier.scala | 122 -
.../scala/io/glutenproject/cbo/rule/Shape.scala | 49 -
.../io/glutenproject/cbo/util/CycleDetector.scala | 63 -
.../glutenproject/cbo/util/IndexDisjointSet.scala | 93 -
.../scala/io/glutenproject/cbo/util/NodeMap.scala | 60 -
.../glutenproject/cbo/vis/GraphvizVisualizer.scala | 214 --
.../src/main/scala/org/apache/gluten/cbo/Cbo.scala | 239 ++
.../scala/org/apache/gluten/cbo/CboCluster.scala | 84 +
.../scala/org/apache/gluten/cbo/CboConfig.scala | 31 +
.../scala/org/apache/gluten/cbo/CboExplain.scala | 21 +
.../scala/org/apache/gluten/cbo/CboGroup.scala | 56 +
.../main/scala/org/apache/gluten/cbo/CboNode.scala | 136 ++
.../scala/org/apache/gluten/cbo/CboPlanner.scala | 154 ++
.../scala/org/apache/gluten/cbo/CostModel.scala | 25 +
.../org/apache/gluten/cbo/MetadataModel.scala | 28 +
.../scala/org/apache/gluten/cbo/PlanModel.scala | 32 +
.../org/apache/gluten/cbo/PropertyModel.scala | 42 +
.../org/apache/gluten/cbo/best/BestFinder.scala | 68 +
.../gluten/cbo/best/GroupBasedBestFinder.scala | 121 +
.../org/apache/gluten/cbo/dp/DpClusterAlgo.scala | 105 +
.../org/apache/gluten/cbo/dp/DpGroupAlgo.scala | 95 +
.../scala/org/apache/gluten/cbo/dp/DpPlanner.scala | 200 ++
.../org/apache/gluten/cbo/dp/DpZipperAlgo.scala | 656 ++++++
.../gluten/cbo/exaustive/ExhaustivePlanner.scala | 141 ++
.../apache/gluten/cbo/memo/ForwardMemoTable.scala | 203 ++
.../scala/org/apache/gluten/cbo/memo/Memo.scala | 227 ++
.../org/apache/gluten/cbo/memo/MemoTable.scala | 91 +
.../scala/org/apache/gluten/cbo/path/CboPath.scala | 195 ++
.../org/apache/gluten/cbo/path/OutputFilter.scala | 116 +
.../org/apache/gluten/cbo/path/OutputWizard.scala | 429 ++++
.../org/apache/gluten/cbo/path/PathFinder.scala | 191 ++
.../scala/org/apache/gluten/cbo/path/PathKey.scala | 60 +
.../org/apache/gluten/cbo/path/PathMask.scala | 192 ++
.../scala/org/apache/gluten/cbo/path/Pattern.scala | 133 ++
.../apache/gluten/cbo/property/PropertySet.scala | 68 +
.../scala/org/apache/gluten/cbo/rule/CboRule.scala | 39 +
.../org/apache/gluten/cbo/rule/EnforcerRule.scala | 93 +
.../org/apache/gluten/cbo/rule/RuleApplier.scala | 122 +
.../scala/org/apache/gluten/cbo/rule/Shape.scala | 49 +
.../org/apache/gluten/cbo/util/CycleDetector.scala | 63 +
.../apache/gluten/cbo/util/IndexDisjointSet.scala | 93 +
.../scala/org/apache/gluten/cbo/util/NodeMap.scala | 60 +
.../apache/gluten/cbo/vis/GraphvizVisualizer.scala | 214 ++
.../io/glutenproject/cbo/CboMetadataSuite.scala | 183 --
.../io/glutenproject/cbo/CboOperationSuite.scala | 469 ----
.../io/glutenproject/cbo/CboPropertySuite.scala | 734 ------
.../test/scala/io/glutenproject/cbo/CboSuite.scala | 483 ----
.../scala/io/glutenproject/cbo/CboSuiteBase.scala | 212 --
.../io/glutenproject/cbo/mock/MockCboPath.scala | 51 -
.../io/glutenproject/cbo/mock/MockMemoState.scala | 161 --
.../io/glutenproject/cbo/path/CboPathSuite.scala | 114 -
.../glutenproject/cbo/path/PathFinderSuite.scala | 311 ---
.../io/glutenproject/cbo/path/PathMaskSuite.scala | 111 -
.../io/glutenproject/cbo/path/WizardSuite.scala | 309 ---
.../io/glutenproject/cbo/rule/PatternSuite.scala | 206 --
.../cbo/specific/CyclicSearchSpaceSuite.scala | 229 --
.../cbo/specific/DistributedSuite.scala | 517 ----
.../cbo/specific/JoinReorderSuite.scala | 433 ----
.../cbo/util/IndexDisjointSetSuite.scala | 96 -
.../org/apache/gluten/cbo/CboMetadataSuite.scala | 183 ++
.../org/apache/gluten/cbo/CboOperationSuite.scala | 469 ++++
.../org/apache/gluten/cbo/CboPropertySuite.scala | 734 ++++++
.../scala/org/apache/gluten/cbo/CboSuite.scala | 483 ++++
.../scala/org/apache/gluten/cbo/CboSuiteBase.scala | 212 ++
.../org/apache/gluten/cbo/mock/MockCboPath.scala | 51 +
.../org/apache/gluten/cbo/mock/MockMemoState.scala | 161 ++
.../org/apache/gluten/cbo/path/CboPathSuite.scala | 114 +
.../apache/gluten/cbo/path/PathFinderSuite.scala | 311 +++
.../org/apache/gluten/cbo/path/PathMaskSuite.scala | 111 +
.../org/apache/gluten/cbo/path/WizardSuite.scala | 309 +++
.../org/apache/gluten/cbo/rule/PatternSuite.scala | 206 ++
.../cbo/specific/CyclicSearchSpaceSuite.scala | 229 ++
.../gluten/cbo/specific/DistributedSuite.scala | 517 ++++
.../gluten/cbo/specific/JoinReorderSuite.scala | 433 ++++
.../gluten/cbo/util/IndexDisjointSetSuite.scala | 96 +
gluten-cbo/planner/pom.xml | 4 +-
.../apache/gluten}/planner/.gitkeep | 0
.../apache/gluten}/planner/.gitkeep | 0
gluten-cbo/pom.xml | 2 +-
gluten-celeborn/clickhouse/pom.xml | 10 +-
.../CHCelebornColumnarBatchSerializer.scala | 8 +-
.../CHCelebornHashBasedColumnarShuffleWriter.scala | 14 +-
...bornHashBasedColumnarShuffleWriterFactory.scala | 2 +-
...lutenClickHouseRSSColumnarShuffleAQESuite.scala | 209 --
...lutenClickHouseRSSColumnarShuffleAQESuite.scala | 209 ++
gluten-celeborn/common/pom.xml | 2 +-
.../gluten/celeborn/CelebornShuffleManager.java | 4 +-
.../CelebornHashBasedColumnarShuffleWriter.scala | 2 +-
gluten-celeborn/package/pom.xml | 6 +-
gluten-celeborn/pom.xml | 4 +-
gluten-celeborn/velox/pom.xml | 8 +-
.../VeloxCelebornColumnarBatchSerializer.scala | 12 +-
...loxCelebornHashBasedColumnarShuffleWriter.scala | 14 +-
...bornHashBasedColumnarShuffleWriterFactory.scala | 2 +-
gluten-core/pom.xml | 12 +-
.../glutenproject/exception/GlutenException.java | 44 -
.../exception/GlutenNotSupportException.java | 39 -
.../glutenproject/memory/MemoryUsageRecorder.java | 25 -
.../memory/MemoryUsageStatsBuilder.java | 24 -
.../memory/SimpleMemoryUsageRecorder.java | 66 -
.../memory/memtarget/KnownNameAndStats.java | 31 -
.../memory/memtarget/LoggingMemoryTarget.java | 66 -
.../memory/memtarget/MemoryTarget.java | 31 -
.../memory/memtarget/MemoryTargetUtil.java | 34 -
.../memory/memtarget/MemoryTargetVisitor.java | 36 -
.../memory/memtarget/MemoryTargets.java | 59 -
.../memory/memtarget/NoopMemoryTarget.java | 57 -
.../memory/memtarget/OverAcquire.java | 95 -
.../io/glutenproject/memory/memtarget/Spiller.java | 31 -
.../glutenproject/memory/memtarget/Spillers.java | 65 -
.../memory/memtarget/ThrowOnOomMemoryTarget.java | 121 -
.../memory/memtarget/TreeMemoryTarget.java | 40 -
.../memory/memtarget/TreeMemoryTargets.java | 231 --
.../memtarget/spark/RegularMemoryConsumer.java | 138 --
.../memory/memtarget/spark/TreeMemoryConsumer.java | 158 --
.../memtarget/spark/TreeMemoryConsumers.java | 96 -
.../java/io/glutenproject/metrics/IMetrics.java | 19 -
.../io/glutenproject/metrics/IOperatorMetrics.java | 19 -
.../java/io/glutenproject/row/SparkRowInfo.java | 35 -
.../glutenproject/substrait/ddlplan/DllNode.java | 39 -
.../substrait/ddlplan/DllPlanNode.java | 29 -
.../substrait/ddlplan/DllTransformContext.scala | 28 -
.../substrait/ddlplan/InsertOutputBuilder.java | 26 -
.../substrait/ddlplan/InsertOutputNode.java | 56 -
.../substrait/ddlplan/InsertPlanNode.java | 50 -
.../substrait/derivation/BinaryOPNode.java | 55 -
.../derivation/DerivationExpressionBuilder.java | 30 -
.../derivation/DerivationExpressionNode.java | 23 -
.../derivation/DerivationFP64TypeNode.java | 44 -
.../expression/AggregateFunctionNode.java | 79 -
.../substrait/expression/BinaryLiteralNode.java | 38 -
.../substrait/expression/BooleanLiteralNode.java | 37 -
.../substrait/expression/ByteLiteralNode.java | 37 -
.../substrait/expression/CastNode.java | 53 -
.../substrait/expression/DateLiteralNode.java | 37 -
.../substrait/expression/DecimalLiteralNode.java | 116 -
.../substrait/expression/DoubleLiteralNode.java | 37 -
.../substrait/expression/ExpressionBuilder.java | 299 ---
.../substrait/expression/ExpressionNode.java | 29 -
.../substrait/expression/FloatLiteralNode.java | 37 -
.../substrait/expression/IfThenNode.java | 63 -
.../substrait/expression/IntLiteralNode.java | 37 -
.../substrait/expression/ListLiteralNode.java | 51 -
.../substrait/expression/LiteralNode.java | 44 -
.../substrait/expression/LiteralNodeWithValue.java | 44 -
.../substrait/expression/LongLiteralNode.java | 37 -
.../substrait/expression/MapLiteralNode.java | 60 -
.../substrait/expression/NullLiteralNode.java | 34 -
.../substrait/expression/ScalarFunctionNode.java | 54 -
.../substrait/expression/SelectionNode.java | 84 -
.../substrait/expression/ShortLiteralNode.java | 37 -
.../substrait/expression/SingularOrListNode.java | 45 -
.../substrait/expression/StringLiteralNode.java | 37 -
.../substrait/expression/StringMapNode.java | 52 -
.../substrait/expression/StructLiteralNode.java | 101 -
.../substrait/expression/TimestampLiteralNode.java | 37 -
.../substrait/expression/WindowFunctionNode.java | 145 --
.../extensions/AdvancedExtensionNode.java | 53 -
.../substrait/extensions/ExtensionBuilder.java | 35 -
.../substrait/extensions/FunctionMappingNode.java | 42 -
.../glutenproject/substrait/plan/PlanBuilder.java | 88 -
.../io/glutenproject/substrait/plan/PlanNode.java | 91 -
.../substrait/rel/AggregateRelNode.java | 98 -
.../glutenproject/substrait/rel/CrossRelNode.java | 72 -
.../glutenproject/substrait/rel/ExpandRelNode.java | 79 -
.../substrait/rel/ExtensionTableBuilder.java | 64 -
.../substrait/rel/ExtensionTableNode.java | 194 --
.../glutenproject/substrait/rel/FetchRelNode.java | 69 -
.../glutenproject/substrait/rel/FilterRelNode.java | 63 -
.../substrait/rel/GenerateRelNode.java | 78 -
.../substrait/rel/InputIteratorRelNode.java | 66 -
.../glutenproject/substrait/rel/JoinRelNode.java | 77 -
.../substrait/rel/LocalFilesBuilder.java | 48 -
.../substrait/rel/LocalFilesNode.java | 226 --
.../substrait/rel/ProjectRelNode.java | 81 -
.../glutenproject/substrait/rel/ReadRelNode.java | 92 -
.../io/glutenproject/substrait/rel/RelBuilder.java | 278 ---
.../io/glutenproject/substrait/rel/RelNode.java | 31 -
.../glutenproject/substrait/rel/SortRelNode.java | 71 -
.../io/glutenproject/substrait/rel/SplitInfo.java | 34 -
.../glutenproject/substrait/rel/WindowRelNode.java | 96 -
.../glutenproject/substrait/rel/WriteRelNode.java | 94 -
.../substrait/type/BinaryTypeNode.java | 43 -
.../substrait/type/BooleanTypeNode.java | 43 -
.../substrait/type/ColumnTypeNode.java | 34 -
.../glutenproject/substrait/type/DateTypeNode.java | 42 -
.../substrait/type/DecimalTypeNode.java | 49 -
.../glutenproject/substrait/type/FP32TypeNode.java | 43 -
.../glutenproject/substrait/type/FP64TypeNode.java | 43 -
.../substrait/type/FixedBinaryTypeNode.java | 46 -
.../substrait/type/FixedCharTypeNode.java | 46 -
.../glutenproject/substrait/type/I16TypeNode.java | 43 -
.../glutenproject/substrait/type/I32TypeNode.java | 43 -
.../glutenproject/substrait/type/I64TypeNode.java | 43 -
.../glutenproject/substrait/type/I8TypeNode.java | 43 -
.../substrait/type/IntervalYearTypeNode.java | 43 -
.../io/glutenproject/substrait/type/ListNode.java | 48 -
.../io/glutenproject/substrait/type/MapNode.java | 64 -
.../glutenproject/substrait/type/NothingNode.java | 33 -
.../substrait/type/StringTypeNode.java | 43 -
.../glutenproject/substrait/type/StructNode.java | 60 -
.../substrait/type/TimestampTypeNode.java | 43 -
.../glutenproject/substrait/type/TypeBuilder.java | 103 -
.../io/glutenproject/substrait/type/TypeNode.java | 23 -
.../main/java/io/glutenproject/test/TestStats.java | 201 --
.../validate/NativePlanValidationInfo.java | 40 -
.../vectorized/GeneralInIterator.java | 40 -
.../vectorized/GeneralOutIterator.java | 72 -
.../io/glutenproject/vectorized/JniLibLoader.java | 328 ---
.../vectorized/JniResourceHelper.java | 205 --
.../io/glutenproject/vectorized/JniWorkspace.java | 144 --
.../vectorized/NativePartitioning.java | 89 -
.../io/glutenproject/vectorized/SplitResult.java | 84 -
.../apache/gluten/exception/GlutenException.java | 44 +
.../exception/GlutenNotSupportException.java | 39 +
.../apache/gluten/memory/MemoryUsageRecorder.java | 25 +
.../gluten/memory/MemoryUsageStatsBuilder.java | 24 +
.../gluten/memory/SimpleMemoryUsageRecorder.java | 66 +
.../gluten/memory/memtarget/KnownNameAndStats.java | 31 +
.../memory/memtarget/LoggingMemoryTarget.java | 66 +
.../gluten/memory/memtarget/MemoryTarget.java | 31 +
.../gluten/memory/memtarget/MemoryTargetUtil.java | 34 +
.../memory/memtarget/MemoryTargetVisitor.java | 36 +
.../gluten/memory/memtarget/MemoryTargets.java | 59 +
.../gluten/memory/memtarget/NoopMemoryTarget.java | 57 +
.../gluten/memory/memtarget/OverAcquire.java | 95 +
.../apache/gluten/memory/memtarget/Spiller.java | 31 +
.../apache/gluten/memory/memtarget/Spillers.java | 65 +
.../memory/memtarget/ThrowOnOomMemoryTarget.java | 121 +
.../gluten/memory/memtarget/TreeMemoryTarget.java | 40 +
.../gluten/memory/memtarget/TreeMemoryTargets.java | 231 ++
.../memtarget/spark/RegularMemoryConsumer.java | 138 ++
.../memory/memtarget/spark/TreeMemoryConsumer.java | 158 ++
.../memtarget/spark/TreeMemoryConsumers.java | 96 +
.../java/org/apache/gluten/metrics/IMetrics.java | 19 +
.../apache/gluten/metrics/IOperatorMetrics.java | 19 +
.../java/org/apache/gluten/row/SparkRowInfo.java | 35 +
.../apache/gluten/substrait/ddlplan/DllNode.java | 39 +
.../gluten/substrait/ddlplan/DllPlanNode.java | 29 +
.../substrait/ddlplan/DllTransformContext.scala | 28 +
.../substrait/ddlplan/InsertOutputBuilder.java | 26 +
.../gluten/substrait/ddlplan/InsertOutputNode.java | 56 +
.../gluten/substrait/ddlplan/InsertPlanNode.java | 50 +
.../gluten/substrait/derivation/BinaryOPNode.java | 55 +
.../derivation/DerivationExpressionBuilder.java | 30 +
.../derivation/DerivationExpressionNode.java | 23 +
.../derivation/DerivationFP64TypeNode.java | 44 +
.../expression/AggregateFunctionNode.java | 79 +
.../substrait/expression/BinaryLiteralNode.java | 38 +
.../substrait/expression/BooleanLiteralNode.java | 37 +
.../substrait/expression/ByteLiteralNode.java | 37 +
.../gluten/substrait/expression/CastNode.java | 53 +
.../substrait/expression/DateLiteralNode.java | 37 +
.../substrait/expression/DecimalLiteralNode.java | 116 +
.../substrait/expression/DoubleLiteralNode.java | 37 +
.../substrait/expression/ExpressionBuilder.java | 299 +++
.../substrait/expression/ExpressionNode.java | 29 +
.../substrait/expression/FloatLiteralNode.java | 37 +
.../gluten/substrait/expression/IfThenNode.java | 63 +
.../substrait/expression/IntLiteralNode.java | 37 +
.../substrait/expression/ListLiteralNode.java | 51 +
.../gluten/substrait/expression/LiteralNode.java | 44 +
.../substrait/expression/LiteralNodeWithValue.java | 44 +
.../substrait/expression/LongLiteralNode.java | 37 +
.../substrait/expression/MapLiteralNode.java | 60 +
.../substrait/expression/NullLiteralNode.java | 34 +
.../substrait/expression/ScalarFunctionNode.java | 54 +
.../gluten/substrait/expression/SelectionNode.java | 84 +
.../substrait/expression/ShortLiteralNode.java | 37 +
.../substrait/expression/SingularOrListNode.java | 45 +
.../substrait/expression/StringLiteralNode.java | 37 +
.../gluten/substrait/expression/StringMapNode.java | 52 +
.../substrait/expression/StructLiteralNode.java | 101 +
.../substrait/expression/TimestampLiteralNode.java | 37 +
.../substrait/expression/WindowFunctionNode.java | 145 ++
.../extensions/AdvancedExtensionNode.java | 53 +
.../substrait/extensions/ExtensionBuilder.java | 35 +
.../substrait/extensions/FunctionMappingNode.java | 42 +
.../apache/gluten/substrait/plan/PlanBuilder.java | 88 +
.../org/apache/gluten/substrait/plan/PlanNode.java | 91 +
.../gluten/substrait/rel/AggregateRelNode.java | 98 +
.../apache/gluten/substrait/rel/CrossRelNode.java | 72 +
.../apache/gluten/substrait/rel/ExpandRelNode.java | 79 +
.../substrait/rel/ExtensionTableBuilder.java | 64 +
.../gluten/substrait/rel/ExtensionTableNode.java | 194 ++
.../apache/gluten/substrait/rel/FetchRelNode.java | 69 +
.../apache/gluten/substrait/rel/FilterRelNode.java | 63 +
.../gluten/substrait/rel/GenerateRelNode.java | 78 +
.../gluten/substrait/rel/InputIteratorRelNode.java | 66 +
.../apache/gluten/substrait/rel/JoinRelNode.java | 77 +
.../gluten/substrait/rel/LocalFilesBuilder.java | 48 +
.../gluten/substrait/rel/LocalFilesNode.java | 226 ++
.../gluten/substrait/rel/ProjectRelNode.java | 81 +
.../apache/gluten/substrait/rel/ReadRelNode.java | 92 +
.../apache/gluten/substrait/rel/RelBuilder.java | 278 +++
.../org/apache/gluten/substrait/rel/RelNode.java | 31 +
.../apache/gluten/substrait/rel/SortRelNode.java | 71 +
.../org/apache/gluten/substrait/rel/SplitInfo.java | 34 +
.../apache/gluten/substrait/rel/WindowRelNode.java | 96 +
.../apache/gluten/substrait/rel/WriteRelNode.java | 94 +
.../gluten/substrait/type/BinaryTypeNode.java | 43 +
.../gluten/substrait/type/BooleanTypeNode.java | 43 +
.../gluten/substrait/type/ColumnTypeNode.java | 34 +
.../apache/gluten/substrait/type/DateTypeNode.java | 42 +
.../gluten/substrait/type/DecimalTypeNode.java | 49 +
.../apache/gluten/substrait/type/FP32TypeNode.java | 43 +
.../apache/gluten/substrait/type/FP64TypeNode.java | 43 +
.../gluten/substrait/type/FixedBinaryTypeNode.java | 46 +
.../gluten/substrait/type/FixedCharTypeNode.java | 46 +
.../apache/gluten/substrait/type/I16TypeNode.java | 43 +
.../apache/gluten/substrait/type/I32TypeNode.java | 43 +
.../apache/gluten/substrait/type/I64TypeNode.java | 43 +
.../apache/gluten/substrait/type/I8TypeNode.java | 43 +
.../substrait/type/IntervalYearTypeNode.java | 43 +
.../org/apache/gluten/substrait/type/ListNode.java | 48 +
.../org/apache/gluten/substrait/type/MapNode.java | 64 +
.../apache/gluten/substrait/type/NothingNode.java | 33 +
.../gluten/substrait/type/StringTypeNode.java | 43 +
.../apache/gluten/substrait/type/StructNode.java | 60 +
.../gluten/substrait/type/TimestampTypeNode.java | 43 +
.../apache/gluten/substrait/type/TypeBuilder.java | 103 +
.../org/apache/gluten/substrait/type/TypeNode.java | 23 +
.../java/org/apache/gluten/test/TestStats.java | 201 ++
.../gluten/validate/NativePlanValidationInfo.java | 40 +
.../gluten/vectorized/GeneralInIterator.java | 40 +
.../gluten/vectorized/GeneralOutIterator.java | 72 +
.../org/apache/gluten/vectorized/JniLibLoader.java | 328 +++
.../gluten/vectorized/JniResourceHelper.java | 205 ++
.../org/apache/gluten/vectorized/JniWorkspace.java | 144 ++
.../gluten/vectorized/NativePartitioning.java | 89 +
.../org/apache/gluten/vectorized/SplitResult.java | 84 +
.../resources/io/glutenproject/proto/config.proto | 11 -
.../resources/io/glutenproject/proto/memory.proto | 17 -
.../resources/org/apache/gluten/proto/config.proto | 11 +
.../resources/org/apache/gluten/proto/memory.proto | 17 +
.../main/scala/io/glutenproject/GlutenPlugin.scala | 281 ---
.../io/glutenproject/backendsapi/Backend.scala | 45 -
.../backendsapi/BackendSettingsApi.scala | 139 --
.../backendsapi/BackendsApiManager.scala | 93 -
.../glutenproject/backendsapi/BroadcastApi.scala | 42 -
.../io/glutenproject/backendsapi/IteratorApi.scala | 94 -
.../io/glutenproject/backendsapi/ListenerApi.scala | 26 -
.../io/glutenproject/backendsapi/MetricsApi.scala | 116 -
.../backendsapi/SparkPlanExecApi.scala | 665 ------
.../glutenproject/backendsapi/TransformerApi.scala | 78 -
.../glutenproject/backendsapi/ValidatorApi.scala | 66 -
.../glutenproject/execution/BaseDataSource.scala | 35 -
.../BasicPhysicalOperatorTransformer.scala | 385 ---
.../execution/BasicScanExecTransformer.scala | 180 --
.../execution/BatchScanExecTransformer.scala | 159 --
.../execution/BroadcastBuildSideRDD.scala | 38 -
.../BroadcastNestedLoopJoinExecTransformer.scala | 191 --
.../CartesianProductExecTransformer.scala | 230 --
.../execution/CoalesceExecTransformer.scala | 76 -
.../execution/ColumnarToRowExecBase.scala | 53 -
.../DataSourceScanTransformerRegister.scala | 59 -
.../execution/DatasourceScanTransformer.scala | 29 -
.../execution/ExpandExecTransformer.scala | 130 -
.../execution/FileSourceScanExecTransformer.scala | 195 --
.../execution/GenerateExecTransformerBase.scala | 98 -
.../execution/GlutenWholeStageColumnarRDD.scala | 141 --
.../HashAggregateExecBaseTransformer.scala | 186 --
.../execution/JoinExecTransformer.scala | 431 ----
.../io/glutenproject/execution/JoinUtils.scala | 361 ---
.../glutenproject/execution/LimitTransformer.scala | 100 -
.../execution/RowToColumnarExecBase.scala | 64 -
.../execution/ScanTransformerFactory.scala | 162 --
.../execution/SortExecTransformer.scala | 137 --
.../execution/SortMergeJoinExecTransformer.scala | 236 --
.../TakeOrderedAndProjectExecTransformer.scala | 160 --
.../execution/WholeStageTransformer.scala | 446 ----
.../execution/WholeStageZippedPartitionsRDD.scala | 74 -
.../execution/WindowExecTransformer.scala | 212 --
.../execution/WriteFilesExecTransformer.scala | 207 --
.../expression/AggregateFunctionsBuilder.scala | 84 -
.../expression/ArrayExpressionTransformer.scala | 95 -
.../expression/BoundReferenceTransformer.scala | 29 -
.../expression/ConditionalTransformer.scala | 67 -
.../glutenproject/expression/ConverterUtils.scala | 458 ----
.../DateTimeExpressionsTransformer.scala | 210 --
.../expression/DecimalRoundTransformer.scala | 77 -
.../expression/ExpressionConverter.scala | 715 ------
.../expression/ExpressionMappings.scala | 316 ---
.../expression/ExpressionTransformer.scala | 23 -
.../glutenproject/expression/ExpressionUtils.scala | 34 -
.../expression/GenericExpressionTransformer.scala | 46 -
.../expression/HashExpressionTransformer.scala | 44 -
.../JsonTupleExpressionTransformer.scala | 55 -
.../expression/LambdaFunctionTransformer.scala | 51 -
.../expression/LiteralTransformer.scala | 28 -
.../expression/MapExpressionTransformer.scala | 91 -
.../expression/NamedExpressionsTransformer.scala | 82 -
.../PredicateExpressionTransformer.scala | 127 -
.../expression/ScalarSubqueryTransformer.scala | 65 -
.../expression/StringExpressionTransformer.scala | 49 -
.../expression/StructExpressionTransformer.scala | 54 -
.../expression/TimestampAddTransform.scala | 53 -
.../expression/TransformerState.scala | 27 -
.../io/glutenproject/expression/UDFMappings.scala | 79 -
.../expression/UnaryExpressionTransformer.scala | 242 --
.../expression/WindowFunctionsBuilder.scala | 67 -
.../extension/ColumnarOverrides.scala | 378 ---
.../CommonSubexpressionEliminateRule.scala | 232 --
.../extension/CountDistinctWithoutExpand.scala | 49 -
.../extension/ExpressionExtensionTrait.scala | 70 -
.../io/glutenproject/extension/GlutenPlan.scala | 108 -
.../extension/OthersExtensionOverrides.scala | 42 -
.../extension/QueryStagePrepOverrides.scala | 50 -
.../glutenproject/extension/RewriteCollect.scala | 141 --
.../RewriteDateTimestampComparisonRule.scala | 374 ---
.../io/glutenproject/extension/RewriteIn.scala | 73 -
.../extension/StrategyOverrides.scala | 221 --
.../columnar/CollapseProjectExecTransformer.scala | 63 -
.../extension/columnar/ColumnarTransitions.scala | 73 -
.../columnar/EnsureLocalSortRequirements.scala | 75 -
.../extension/columnar/EnumeratedTransform.scala | 66 -
.../extension/columnar/ExpandFallbackPolicy.scala | 294 ---
.../columnar/MergeTwoPhasesHashAggregate.scala | 130 -
.../extension/columnar/MiscColumnarRules.scala | 126 -
.../extension/columnar/PullOutPostProject.scala | 112 -
.../extension/columnar/PullOutPreProject.scala | 199 --
.../RemoveNativeWriteFilesSortAndProject.scala | 121 -
.../columnar/RewriteMultiChildrenCount.scala | 108 -
.../columnar/RewriteSparkPlanRulesManager.scala | 127 -
.../extension/columnar/RewriteTransformer.scala | 53 -
.../columnar/RewriteTypedImperativeAggregate.scala | 73 -
.../extension/columnar/TransformHintRule.scala | 758 ------
.../columnar/transform/ImplementSingleNode.scala | 517 ----
.../scala/io/glutenproject/glutenproject.scala | 78 -
.../io/glutenproject/metrics/MetricsUpdater.scala | 41 -
.../glutenproject/planner/GlutenOptimization.scala | 53 -
.../io/glutenproject/planner/cost/GlutenCost.scala | 21 -
.../planner/cost/GlutenCostModel.scala | 73 -
.../planner/metadata/GlutenMetadata.scala | 35 -
.../planner/metadata/GlutenMetadataModel.scala | 48 -
.../planner/plan/GlutenPlanModel.scala | 80 -
.../planner/property/GlutenPropertyModel.scala | 133 --
.../glutenproject/planner/rule/GlutenRules.scala | 27 -
.../softaffinity/SoftAffinityManager.scala | 307 ---
.../scheduler/SoftAffinityListener.scala | 48 -
.../strategy/SoftAffinityAllocationTrait.scala | 36 -
.../strategy/SoftAffinityStrategy.scala | 53 -
.../glutenproject/substrait/SubstraitContext.scala | 194 --
.../scala/io/glutenproject/utils/DebugUtil.scala | 44 -
.../utils/DecimalArithmeticUtil.scala | 292 ---
.../io/glutenproject/utils/FallbackUtil.scala | 84 -
.../glutenproject/utils/InputPartitionsUtil.scala | 116 -
.../scala/io/glutenproject/utils/Iterators.scala | 202 --
.../io/glutenproject/utils/LogLevelUtil.scala | 42 -
.../scala/io/glutenproject/utils/PlanUtil.scala | 80 -
.../glutenproject/utils/PullOutProjectHelper.scala | 156 --
.../io/glutenproject/utils/QueryPlanSelector.scala | 85 -
.../utils/SubstraitPlanPrinterUtil.scala | 66 -
.../io/glutenproject/utils/SubstraitUtil.scala | 54 -
.../io/glutenproject/utils/TaskListener.scala | 25 -
.../scala/org/apache/gluten/GlutenPlugin.scala | 281 +++
.../org/apache/gluten/backendsapi/Backend.scala | 45 +
.../gluten/backendsapi/BackendSettingsApi.scala | 139 ++
.../gluten/backendsapi/BackendsApiManager.scala | 93 +
.../apache/gluten/backendsapi/BroadcastApi.scala | 42 +
.../apache/gluten/backendsapi/IteratorApi.scala | 94 +
.../apache/gluten/backendsapi/ListenerApi.scala | 26 +
.../org/apache/gluten/backendsapi/MetricsApi.scala | 116 +
.../gluten/backendsapi/SparkPlanExecApi.scala | 665 ++++++
.../apache/gluten/backendsapi/TransformerApi.scala | 78 +
.../apache/gluten/backendsapi/ValidatorApi.scala | 66 +
.../apache/gluten/execution/BaseDataSource.scala | 35 +
.../BasicPhysicalOperatorTransformer.scala | 385 +++
.../execution/BasicScanExecTransformer.scala | 180 ++
.../execution/BatchScanExecTransformer.scala | 159 ++
.../gluten/execution/BroadcastBuildSideRDD.scala | 38 +
.../BroadcastNestedLoopJoinExecTransformer.scala | 191 ++
.../CartesianProductExecTransformer.scala | 230 ++
.../gluten/execution/CoalesceExecTransformer.scala | 76 +
.../gluten/execution/ColumnarToRowExecBase.scala | 53 +
.../DataSourceScanTransformerRegister.scala | 59 +
.../execution/DatasourceScanTransformer.scala | 29 +
.../gluten/execution/ExpandExecTransformer.scala | 130 +
.../execution/FileSourceScanExecTransformer.scala | 195 ++
.../execution/GenerateExecTransformerBase.scala | 98 +
.../execution/GlutenWholeStageColumnarRDD.scala | 141 ++
.../HashAggregateExecBaseTransformer.scala | 186 ++
.../gluten/execution/JoinExecTransformer.scala | 431 ++++
.../org/apache/gluten/execution/JoinUtils.scala | 361 +++
.../apache/gluten/execution/LimitTransformer.scala | 100 +
.../gluten/execution/RowToColumnarExecBase.scala | 64 +
.../gluten/execution/ScanTransformerFactory.scala | 162 ++
.../gluten/execution/SortExecTransformer.scala | 137 ++
.../execution/SortMergeJoinExecTransformer.scala | 236 ++
.../TakeOrderedAndProjectExecTransformer.scala | 160 ++
.../gluten/execution/WholeStageTransformer.scala | 446 ++++
.../execution/WholeStageZippedPartitionsRDD.scala | 74 +
.../gluten/execution/WindowExecTransformer.scala | 212 ++
.../execution/WriteFilesExecTransformer.scala | 207 ++
.../expression/AggregateFunctionsBuilder.scala | 84 +
.../expression/ArrayExpressionTransformer.scala | 95 +
.../expression/BoundReferenceTransformer.scala | 29 +
.../gluten/expression/ConditionalTransformer.scala | 67 +
.../apache/gluten/expression/ConverterUtils.scala | 458 ++++
.../DateTimeExpressionsTransformer.scala | 210 ++
.../expression/DecimalRoundTransformer.scala | 77 +
.../gluten/expression/ExpressionConverter.scala | 715 ++++++
.../gluten/expression/ExpressionMappings.scala | 316 +++
.../gluten/expression/ExpressionTransformer.scala | 23 +
.../apache/gluten/expression/ExpressionUtils.scala | 34 +
.../expression/GenericExpressionTransformer.scala | 46 +
.../expression/HashExpressionTransformer.scala | 44 +
.../JsonTupleExpressionTransformer.scala | 55 +
.../expression/LambdaFunctionTransformer.scala | 51 +
.../gluten/expression/LiteralTransformer.scala | 28 +
.../expression/MapExpressionTransformer.scala | 91 +
.../expression/NamedExpressionsTransformer.scala | 82 +
.../PredicateExpressionTransformer.scala | 127 +
.../expression/ScalarSubqueryTransformer.scala | 65 +
.../expression/StringExpressionTransformer.scala | 49 +
.../expression/StructExpressionTransformer.scala | 54 +
.../gluten/expression/TimestampAddTransform.scala | 53 +
.../gluten/expression/TransformerState.scala | 27 +
.../org/apache/gluten/expression/UDFMappings.scala | 79 +
.../expression/UnaryExpressionTransformer.scala | 242 ++
.../gluten/expression/WindowFunctionsBuilder.scala | 67 +
.../gluten/extension/ColumnarOverrides.scala | 378 +++
.../CommonSubexpressionEliminateRule.scala | 232 ++
.../extension/CountDistinctWithoutExpand.scala | 49 +
.../extension/ExpressionExtensionTrait.scala | 70 +
.../org/apache/gluten/extension/GlutenPlan.scala | 108 +
.../extension/OthersExtensionOverrides.scala | 42 +
.../gluten/extension/QueryStagePrepOverrides.scala | 50 +
.../apache/gluten/extension/RewriteCollect.scala | 141 ++
.../RewriteDateTimestampComparisonRule.scala | 374 +++
.../org/apache/gluten/extension/RewriteIn.scala | 73 +
.../gluten/extension/StrategyOverrides.scala | 221 ++
.../columnar/CollapseProjectExecTransformer.scala | 63 +
.../extension/columnar/ColumnarTransitions.scala | 73 +
.../columnar/EnsureLocalSortRequirements.scala | 75 +
.../extension/columnar/EnumeratedTransform.scala | 66 +
.../extension/columnar/ExpandFallbackPolicy.scala | 294 +++
.../columnar/MergeTwoPhasesHashAggregate.scala | 130 +
.../extension/columnar/MiscColumnarRules.scala | 126 +
.../extension/columnar/PullOutPostProject.scala | 112 +
.../extension/columnar/PullOutPreProject.scala | 199 ++
.../RemoveNativeWriteFilesSortAndProject.scala | 121 +
.../columnar/RewriteMultiChildrenCount.scala | 108 +
.../columnar/RewriteSparkPlanRulesManager.scala | 127 +
.../extension/columnar/RewriteTransformer.scala | 53 +
.../columnar/RewriteTypedImperativeAggregate.scala | 73 +
.../extension/columnar/TransformHintRule.scala | 758 ++++++
.../columnar/transform/ImplementSingleNode.scala | 517 ++++
.../src/main/scala/org/apache/gluten/gluten.scala | 78 +
.../org/apache/gluten/metrics/MetricsUpdater.scala | 41 +
.../apache/gluten/planner/GlutenOptimization.scala | 53 +
.../apache/gluten/planner/cost/GlutenCost.scala | 21 +
.../gluten/planner/cost/GlutenCostModel.scala | 73 +
.../gluten/planner/metadata/GlutenMetadata.scala | 35 +
.../planner/metadata/GlutenMetadataModel.scala | 48 +
.../gluten/planner/plan/GlutenPlanModel.scala | 80 +
.../planner/property/GlutenPropertyModel.scala | 133 ++
.../apache/gluten/planner/rule/GlutenRules.scala | 27 +
.../gluten/softaffinity/SoftAffinityManager.scala | 307 +++
.../scheduler/SoftAffinityListener.scala | 48 +
.../strategy/SoftAffinityAllocationTrait.scala | 36 +
.../strategy/SoftAffinityStrategy.scala | 53 +
.../apache/gluten/substrait/SubstraitContext.scala | 194 ++
.../scala/org/apache/gluten/utils/DebugUtil.scala | 44 +
.../gluten/utils/DecimalArithmeticUtil.scala | 292 +++
.../org/apache/gluten/utils/FallbackUtil.scala | 84 +
.../apache/gluten/utils/InputPartitionsUtil.scala | 116 +
.../scala/org/apache/gluten/utils/Iterators.scala | 202 ++
.../org/apache/gluten/utils/LogLevelUtil.scala | 42 +
.../scala/org/apache/gluten/utils/PlanUtil.scala | 80 +
.../apache/gluten/utils/PullOutProjectHelper.scala | 156 ++
.../apache/gluten/utils/QueryPlanSelector.scala | 85 +
.../gluten/utils/SubstraitPlanPrinterUtil.scala | 66 +
.../org/apache/gluten/utils/SubstraitUtil.scala | 54 +
.../org/apache/gluten/utils/TaskListener.scala | 25 +
.../spark/listener/GlutenListenerFactory.scala | 4 +-
.../org/apache/spark/memory/SparkMemoryUtil.scala | 6 +-
.../apache/spark/rpc/GlutenDriverEndpoint.scala | 2 +-
.../apache/spark/rpc/GlutenExecutorEndpoint.scala | 2 +-
.../spark/shuffle/ColumnarShuffleDependency.scala | 2 +-
.../apache/spark/shuffle/GlutenShuffleUtils.scala | 8 +-
.../spark/shuffle/GlutenShuffleWriterWrapper.scala | 2 +-
.../apache/spark/softaffinity/SoftAffinity.scala | 4 +-
.../execution/ColumnarBroadcastExchangeExec.scala | 8 +-
.../ColumnarCollapseTransformStages.scala | 12 +-
.../execution/ColumnarShuffleExchangeExec.scala | 8 +-
.../execution/ColumnarSubqueryBroadcastExec.scala | 6 +-
.../spark/sql/execution/GlutenExplainUtils.scala | 6 +-
.../sql/execution/GlutenFallbackReporter.scala | 10 +-
.../spark/sql/execution/GlutenImplicits.scala | 6 +-
.../GlutenFormatWriterInjectsBase.scala | 10 +-
.../datasources/GlutenWriterColumnarRules.scala | 6 +-
.../python/EvalPythonExecTransformer.scala | 20 +-
.../spark/sql/hive/HivePartitionConverter.scala | 4 +-
.../sql/hive/HiveTableScanExecTransformer.scala | 10 +-
.../apache/spark/sql/hive/HiveUDFTransformer.scala | 4 +-
.../apache/spark/sql/utils/ExpressionUtil.scala | 2 +-
.../org/apache/spark/util/ExecutorManager.scala | 2 +-
.../org/apache/spark/util/SparkDirectoryUtil.scala | 2 +-
.../org/apache/spark/util/TaskResources.scala | 6 +-
.../benchmarks/RandomParquetDataGenerator.scala | 181 --
.../execution/WholeStageTransformerSuite.scala | 366 ---
.../io/glutenproject/utils/IteratorSuite.scala | 135 --
.../io/glutenproject/utils/TaskResourceSuite.scala | 60 -
.../benchmarks/RandomParquetDataGenerator.scala | 181 ++
.../execution/WholeStageTransformerSuite.scala | 366 +++
.../org/apache/gluten/utils/IteratorSuite.scala | 135 ++
.../apache/gluten/utils/TaskResourceSuite.scala | 60 +
.../spark/softaffinity/SoftAffinitySuite.scala | 12 +-
.../SoftAffinityWithRDDInfoSuite.scala | 12 +-
.../org/apache/spark/sql/GlutenQueryTest.scala | 2 +-
.../scala/org/apache/spark/sql/TestUtils.scala | 2 +-
gluten-data/pom.xml | 6 +-
.../columnarbatch/ColumnarBatchJniWrapper.java | 65 -
.../columnarbatch/ColumnarBatches.java | 403 ----
.../columnarbatch/IndicatorVector.java | 163 --
.../columnarbatch/PlaceholderVector.java | 117 -
.../datasource/DatasourceJniWrapper.java | 64 -
.../exception/JniExceptionDescriber.java | 31 -
.../java/io/glutenproject/exec/RuntimeAware.java | 25 -
.../io/glutenproject/exec/RuntimeJniWrapper.java | 26 -
.../main/java/io/glutenproject/init/JniUtils.java | 30 -
.../init/NativeBackendInitializer.java | 47 -
.../memory/alloc/NativeMemoryAllocator.java | 49 -
.../memory/alloc/NativeMemoryAllocators.java | 54 -
.../memory/arrowalloc/ArrowBufferAllocators.java | 115 -
.../arrowalloc/ManagedAllocationListener.java | 105 -
.../memory/nmm/ManagedReservationListener.java | 67 -
.../memory/nmm/NativeMemoryManager.java | 131 -
.../memory/nmm/NativeMemoryManagers.java | 158 --
.../memory/nmm/ReservationListener.java | 31 -
.../java/io/glutenproject/metrics/Metrics.java | 168 --
.../io/glutenproject/metrics/OperatorMetrics.java | 116 -
.../vectorized/ArrowColumnVector.java | 578 -----
.../vectorized/ArrowWritableColumnVector.java | 1929 ---------------
.../vectorized/ColumnarBatchInIterator.java | 42 -
.../vectorized/ColumnarBatchOutIterator.java | 92 -
.../vectorized/ColumnarBatchSerializeResult.java | 41 -
.../ColumnarBatchSerializerJniWrapper.java | 51 -
.../vectorized/GlutenSplitResult.java | 47 -
.../vectorized/JniByteInputStream.java | 33 -
.../vectorized/JniByteInputStreams.java | 75 -
.../LowCopyFileSegmentJniByteInputStream.java | 128 -
.../vectorized/LowCopyNettyJniByteInputStream.java | 107 -
.../vectorized/NativeColumnarToRowInfo.java | 29 -
.../vectorized/NativeColumnarToRowJniWrapper.java | 45 -
.../vectorized/NativePlanEvaluator.java | 116 -
.../vectorized/NativeRowToColumnarJniWrapper.java | 45 -
.../vectorized/OnHeapJniByteInputStream.java | 66 -
.../vectorized/PlanEvaluatorJniWrapper.java | 74 -
.../vectorized/ShuffleReaderJniWrapper.java | 51 -
.../vectorized/ShuffleReaderMetrics.java | 47 -
.../vectorized/ShuffleWriterJniWrapper.java | 207 --
.../columnarbatch/ColumnarBatchJniWrapper.java | 65 +
.../gluten/columnarbatch/ColumnarBatches.java | 403 ++++
.../gluten/columnarbatch/IndicatorVector.java | 163 ++
.../gluten/columnarbatch/PlaceholderVector.java | 117 +
.../gluten/datasource/DatasourceJniWrapper.java | 64 +
.../gluten/exception/JniExceptionDescriber.java | 31 +
.../java/org/apache/gluten/exec/RuntimeAware.java | 25 +
.../org/apache/gluten/exec/RuntimeJniWrapper.java | 26 +
.../main/java/org/apache/gluten/init/JniUtils.java | 30 +
.../gluten/init/NativeBackendInitializer.java | 47 +
.../gluten/memory/alloc/NativeMemoryAllocator.java | 49 +
.../memory/alloc/NativeMemoryAllocators.java | 54 +
.../memory/arrowalloc/ArrowBufferAllocators.java | 115 +
.../arrowalloc/ManagedAllocationListener.java | 105 +
.../memory/nmm/ManagedReservationListener.java | 67 +
.../gluten/memory/nmm/NativeMemoryManager.java | 131 +
.../gluten/memory/nmm/NativeMemoryManagers.java | 158 ++
.../gluten/memory/nmm/ReservationListener.java | 31 +
.../java/org/apache/gluten/metrics/Metrics.java | 168 ++
.../org/apache/gluten/metrics/OperatorMetrics.java | 116 +
.../gluten/vectorized/ArrowColumnVector.java | 578 +++++
.../vectorized/ArrowWritableColumnVector.java | 1929 +++++++++++++++
.../gluten/vectorized/ColumnarBatchInIterator.java | 42 +
.../vectorized/ColumnarBatchOutIterator.java | 92 +
.../vectorized/ColumnarBatchSerializeResult.java | 41 +
.../ColumnarBatchSerializerJniWrapper.java | 51 +
.../gluten/vectorized/GlutenSplitResult.java | 47 +
.../gluten/vectorized/JniByteInputStream.java | 33 +
.../gluten/vectorized/JniByteInputStreams.java | 75 +
.../LowCopyFileSegmentJniByteInputStream.java | 128 +
.../vectorized/LowCopyNettyJniByteInputStream.java | 107 +
.../gluten/vectorized/NativeColumnarToRowInfo.java | 29 +
.../vectorized/NativeColumnarToRowJniWrapper.java | 45 +
.../gluten/vectorized/NativePlanEvaluator.java | 116 +
.../vectorized/NativeRowToColumnarJniWrapper.java | 45 +
.../vectorized/OnHeapJniByteInputStream.java | 66 +
.../gluten/vectorized/PlanEvaluatorJniWrapper.java | 74 +
.../gluten/vectorized/ShuffleReaderJniWrapper.java | 51 +
.../gluten/vectorized/ShuffleReaderMetrics.java | 47 +
.../gluten/vectorized/ShuffleWriterJniWrapper.java | 207 ++
.../main/scala/io/glutenproject/exec/Runtime.scala | 53 -
.../scala/io/glutenproject/exec/Runtimes.scala | 36 -
.../metrics/BatchScanMetricsUpdater.scala | 56 -
.../metrics/ExpandMetricsUpdater.scala | 35 -
.../metrics/FileSourceScanMetricsUpdater.scala | 79 -
.../metrics/FilterMetricsUpdater.scala | 35 -
.../metrics/GenerateMetricsUpdater.scala | 28 -
.../metrics/HashAggregateMetricsUpdater.scala | 85 -
.../metrics/HiveTableScanMetricsUpdater.scala | 74 -
.../metrics/InputIteratorMetricsUpdater.scala | 37 -
.../glutenproject/metrics/JoinMetricsUpdater.scala | 191 --
.../metrics/LimitMetricsUpdater.scala | 35 -
.../io/glutenproject/metrics/MetricsUtil.scala | 313 ---
.../metrics/NestedLoopJoinMetricsUpdater.scala | 35 -
.../metrics/ProjectMetricsUpdater.scala | 35 -
.../glutenproject/metrics/SortMetricsUpdater.scala | 39 -
.../metrics/WindowMetricsUpdater.scala | 35 -
.../metrics/WriteFilesMetricsUpdater.scala | 30 -
.../io/glutenproject/utils/ArrowAbiUtil.scala | 193 --
.../scala/io/glutenproject/utils/ArrowUtil.scala | 89 -
.../io/glutenproject/utils/ImplicitClass.scala | 38 -
.../vectorized/ColumnarBatchSerializer.scala | 251 --
.../scala/org/apache/gluten/exec/Runtime.scala | 53 +
.../scala/org/apache/gluten/exec/Runtimes.scala | 36 +
.../gluten/metrics/BatchScanMetricsUpdater.scala | 56 +
.../gluten/metrics/ExpandMetricsUpdater.scala | 35 +
.../metrics/FileSourceScanMetricsUpdater.scala | 79 +
.../gluten/metrics/FilterMetricsUpdater.scala | 35 +
.../gluten/metrics/GenerateMetricsUpdater.scala | 28 +
.../metrics/HashAggregateMetricsUpdater.scala | 85 +
.../metrics/HiveTableScanMetricsUpdater.scala | 74 +
.../metrics/InputIteratorMetricsUpdater.scala | 37 +
.../apache/gluten/metrics/JoinMetricsUpdater.scala | 191 ++
.../gluten/metrics/LimitMetricsUpdater.scala | 35 +
.../org/apache/gluten/metrics/MetricsUtil.scala | 313 +++
.../metrics/NestedLoopJoinMetricsUpdater.scala | 35 +
.../gluten/metrics/ProjectMetricsUpdater.scala | 35 +
.../apache/gluten/metrics/SortMetricsUpdater.scala | 39 +
.../gluten/metrics/WindowMetricsUpdater.scala | 35 +
.../gluten/metrics/WriteFilesMetricsUpdater.scala | 30 +
.../org/apache/gluten/utils/ArrowAbiUtil.scala | 193 ++
.../scala/org/apache/gluten/utils/ArrowUtil.scala | 89 +
.../org/apache/gluten/utils/ImplicitClass.scala | 38 +
.../vectorized/ColumnarBatchSerializer.scala | 251 ++
.../spark/shuffle/ColumnarShuffleWriter.scala | 14 +-
.../sql/execution/ColumnarBuildSideRelation.scala | 14 +-
.../spark/sql/execution/utils/ExecUtil.scala | 10 +-
.../apache/spark/sql/utils/SparkVectorUtil.scala | 6 +-
gluten-delta/pom.xml | 10 +-
...ect.execution.DataSourceScanTransformerRegister | 1 -
...ject.extension.columnar.RewriteTransformerRules | 1 -
...ten.execution.DataSourceScanTransformerRegister | 1 +
...uten.extension.columnar.RewriteTransformerRules | 1 +
.../execution/DeltaScanTransformer.scala | 97 -
.../execution/DeltaScanTransformerProvider.scala | 31 -
.../extension/DeltaRewriteTransformerRules.scala | 149 --
.../gluten/execution/DeltaScanTransformer.scala | 97 +
.../execution/DeltaScanTransformerProvider.scala | 31 +
.../extension/DeltaRewriteTransformerRules.scala | 149 ++
.../glutenproject/execution/VeloxDeltaSuite.scala | 214 --
.../execution/VeloxTPCHDeltaSuite.scala | 60 -
.../apache/gluten/execution/VeloxDeltaSuite.scala | 214 ++
.../gluten/execution/VeloxTPCHDeltaSuite.scala | 60 +
gluten-iceberg/pom.xml | 10 +-
.../substrait/rel/IcebergLocalFilesBuilder.java | 44 -
.../substrait/rel/IcebergLocalFilesNode.java | 120 -
.../substrait/rel/IcebergLocalFilesBuilder.java | 44 +
.../substrait/rel/IcebergLocalFilesNode.java | 120 +
...ect.execution.DataSourceScanTransformerRegister | 1 -
...ten.execution.DataSourceScanTransformerRegister | 1 +
.../execution/IcebergScanTransformer.scala | 95 -
.../execution/IcebergTransformerProvider.scala | 31 -
.../gluten/execution/IcebergScanTransformer.scala | 95 +
.../execution/IcebergTransformerProvider.scala | 31 +
.../spark/source/GlutenIcebergSourceUtil.scala | 4 +-
.../execution/VeloxIcebergSuite.scala | 478 ----
.../execution/VeloxTPCHIcebergSuite.scala | 111 -
.../gluten/execution/VeloxIcebergSuite.scala | 478 ++++
.../gluten/execution/VeloxTPCHIcebergSuite.scala | 111 +
gluten-ui/pom.xml | 2 +-
.../io/glutenproject/events/GlutenEvent.scala | 49 -
.../org/apache/gluten/events/GlutenEvent.scala | 49 +
.../spark/sql/execution/ui/GlutenEventUtils.scala | 2 +-
.../execution/ui/GlutenSQLAppStatusListener.scala | 2 +-
gluten-ut/common/pom.xml | 4 +-
.../glutenproject/utils/BackendTestSettings.scala | 226 --
.../io/glutenproject/utils/BackendTestUtils.scala | 33 -
.../glutenproject/utils/SQLQueryTestSettings.scala | 23 -
.../io/glutenproject/utils/SystemParameters.scala | 38 -
.../apache/gluten/utils/BackendTestSettings.scala | 226 ++
.../org/apache/gluten/utils/BackendTestUtils.scala | 33 +
.../apache/gluten/utils/SQLQueryTestSettings.scala | 23 +
.../org/apache/gluten/utils/SystemParameters.scala | 38 +
.../spark/sql/DummyFilterColmnarHelper.scala | 8 +-
.../apache/spark/sql/GlutenSQLTestsBaseTrait.scala | 6 +-
.../org/apache/spark/sql/GlutenSQLTestsTrait.scala | 2 +-
.../apache/spark/sql/GlutenTestsBaseTrait.scala | 2 +-
.../apache/spark/sql/GlutenTestsCommonTrait.scala | 2 +-
.../org/apache/spark/sql/GlutenTestsTrait.scala | 12 +-
gluten-ut/pom.xml | 4 +-
gluten-ut/spark32/pom.xml | 8 +-
.../ClickHouseSQLQueryTestSettings.scala | 25 -
.../utils/clickhouse/ClickHouseTestSettings.scala | 2192 -----------------
.../utils/velox/VeloxSQLQueryTestSettings.scala | 240 --
.../utils/velox/VeloxTestSettings.scala | 1078 ---------
.../ClickHouseSQLQueryTestSettings.scala | 25 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 2197 +++++++++++++++++
.../utils/velox/VeloxSQLQueryTestSettings.scala | 240 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 1078 +++++++++
.../spark/sql/GlutenDataFrameAggregateSuite.scala | 2 +-
.../apache/spark/sql/GlutenDataFrameSuite.scala | 2 +-
.../sql/GlutenDynamicPartitionPruningSuite.scala | 8 +-
.../org/apache/spark/sql/GlutenSQLQuerySuite.scala | 2 +-
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 6 +-
.../sql/GlutenSparkSessionExtensionSuite.scala | 2 +-
.../org/apache/spark/sql/GlutenSubquerySuite.scala | 2 +-
.../sql/execution/FallbackStrategiesSuite.scala | 8 +-
.../GlutenCoalesceShufflePartitionsSuite.scala | 2 +-
.../GlutenSQLAggregateFunctionSuite.scala | 2 +-
.../execution/GlutenSQLWindowFunctionSuite.scala | 2 +-
.../spark/sql/execution/GlutenSortSuite.scala | 2 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 2 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 2 +-
.../benchmarks/ParquetReadBenchmark.scala | 12 +-
.../GlutenWriterColumnarRulesSuite.scala | 2 +-
.../orc/GlutenOrcV1SchemaPruningSuite.scala | 2 +-
.../orc/GlutenOrcV2SchemaPruningSuite.scala | 2 +-
.../parquet/GlutenParquetSchemaPruningSuite.scala | 2 +-
.../execution/joins/GlutenBroadcastJoinSuite.scala | 8 +-
.../extension/CustomerExpressionTransformer.scala | 8 +-
...GlutenCollapseProjectExecTransformerSuite.scala | 4 +-
.../GlutenCustomerExpressionTransformerSuite.scala | 4 +-
.../extension/GlutenSessionExtensionSuite.scala | 2 +-
.../TestFileSourceScanExecTransformer.scala | 4 +-
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 4 +-
.../sql/statistics/SparkFunctionStatistics.scala | 8 +-
gluten-ut/spark33/pom.xml | 8 +-
.../ClickHouseSQLQueryTestSettings.scala | 25 -
.../utils/clickhouse/ClickHouseTestSettings.scala | 2104 -----------------
.../utils/velox/VeloxSQLQueryTestSettings.scala | 243 --
.../utils/velox/VeloxTestSettings.scala | 1132 ---------
.../ClickHouseSQLQueryTestSettings.scala | 25 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 2109 +++++++++++++++++
.../utils/velox/VeloxSQLQueryTestSettings.scala | 243 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 1132 +++++++++
.../sql/GlutenBloomFilterAggregateQuerySuite.scala | 6 +-
.../apache/spark/sql/GlutenCachedTableSuite.scala | 2 +-
.../spark/sql/GlutenDataFrameAggregateSuite.scala | 2 +-
.../apache/spark/sql/GlutenDataFrameSuite.scala | 2 +-
.../sql/GlutenDataFrameWindowFunctionsSuite.scala | 2 +-
.../sql/GlutenDynamicPartitionPruningSuite.scala | 8 +-
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 6 +-
.../sql/GlutenSparkSessionExtensionSuite.scala | 2 +-
.../spark/sql/GlutenStringFunctionsSuite.scala | 2 +-
.../org/apache/spark/sql/GlutenSubquerySuite.scala | 2 +-
.../GlutenKeyGroupedPartitioningSuite.scala | 2 +-
.../sql/execution/FallbackStrategiesSuite.scala | 10 +-
.../GlutenCoalesceShufflePartitionsSuite.scala | 2 +-
.../GlutenReplaceHashWithSortAggSuite.scala | 4 +-
.../GlutenSQLAggregateFunctionSuite.scala | 2 +-
.../execution/GlutenSQLWindowFunctionSuite.scala | 2 +-
.../spark/sql/execution/GlutenSortSuite.scala | 2 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 2 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 2 +-
.../benchmarks/ParquetReadBenchmark.scala | 12 +-
.../GlutenFileMetadataStructSuite.scala | 4 +-
.../GlutenWriterColumnarRulesSuite.scala | 2 +-
.../orc/GlutenOrcV1SchemaPruningSuite.scala | 2 +-
.../orc/GlutenOrcV2SchemaPruningSuite.scala | 2 +-
.../parquet/GlutenParquetSchemaPruningSuite.scala | 2 +-
.../execution/joins/GlutenBroadcastJoinSuite.scala | 6 +-
...GlutenCollapseProjectExecTransformerSuite.scala | 4 +-
.../extension/GlutenSessionExtensionSuite.scala | 2 +-
.../TestFileSourceScanExecTransformer.scala | 4 +-
.../spark/sql/gluten/GlutenFallbackSuite.scala | 8 +-
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 4 +-
.../sql/statistics/SparkFunctionStatistics.scala | 8 +-
gluten-ut/spark34/pom.xml | 8 +-
.../ClickHouseSQLQueryTestSettings.scala | 25 -
.../utils/clickhouse/ClickHouseTestSettings.scala | 1884 ---------------
.../utils/velox/VeloxSQLQueryTestSettings.scala | 247 --
.../utils/velox/VeloxTestSettings.scala | 1143 ---------
.../ClickHouseSQLQueryTestSettings.scala | 25 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 1889 +++++++++++++++
.../utils/velox/VeloxSQLQueryTestSettings.scala | 247 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 1143 +++++++++
.../sql/GlutenBloomFilterAggregateQuerySuite.scala | 6 +-
.../apache/spark/sql/GlutenCachedTableSuite.scala | 2 +-
.../spark/sql/GlutenDataFrameAggregateSuite.scala | 2 +-
.../apache/spark/sql/GlutenDataFrameSuite.scala | 2 +-
.../sql/GlutenDataFrameWindowFunctionsSuite.scala | 2 +-
.../sql/GlutenDynamicPartitionPruningSuite.scala | 6 +-
.../spark/sql/GlutenFileBasedDataSourceSuite.scala | 2 +-
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 6 +-
.../sql/GlutenSparkSessionExtensionSuite.scala | 2 +-
.../spark/sql/GlutenStringFunctionsSuite.scala | 2 +-
.../org/apache/spark/sql/GlutenSubquerySuite.scala | 2 +-
.../sql/connector/GlutenDataSourceV2Suite.scala | 2 +-
.../GlutenKeyGroupedPartitioningSuite.scala | 2 +-
.../sql/execution/FallbackStrategiesSuite.scala | 10 +-
.../GlutenCoalesceShufflePartitionsSuite.scala | 2 +-
.../GlutenReplaceHashWithSortAggSuite.scala | 2 +-
.../GlutenSQLAggregateFunctionSuite.scala | 2 +-
.../execution/GlutenSQLWindowFunctionSuite.scala | 2 +-
.../spark/sql/execution/GlutenSortSuite.scala | 2 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 2 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 2 +-
.../benchmarks/ParquetReadBenchmark.scala | 12 +-
.../GlutenFileMetadataStructSuite.scala | 4 +-
.../datasources/GlutenV1WriteCommandSuite.scala | 2 +-
.../execution/datasources/csv/GlutenCSVSuite.scala | 2 +-
.../orc/GlutenOrcV1SchemaPruningSuite.scala | 2 +-
.../orc/GlutenOrcV2SchemaPruningSuite.scala | 2 +-
.../parquet/GlutenParquetSchemaPruningSuite.scala | 2 +-
.../execution/joins/GlutenBroadcastJoinSuite.scala | 6 +-
...GlutenCollapseProjectExecTransformerSuite.scala | 4 +-
.../extension/GlutenSessionExtensionSuite.scala | 2 +-
.../TestFileSourceScanExecTransformer.scala | 4 +-
.../spark/sql/gluten/GlutenFallbackSuite.scala | 8 +-
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 4 +-
.../spark/sql/sources/GlutenInsertSuite.scala | 4 +-
.../sql/statistics/SparkFunctionStatistics.scala | 8 +-
gluten-ut/test/pom.xml | 10 +-
.../expressions/GlutenExpressionMappingSuite.scala | 97 -
.../expressions/GlutenExpressionMappingSuite.scala | 97 +
package/pom.xml | 8 +-
pom.xml | 6 +-
shims/common/pom.xml | 2 +-
.../main/scala/io/glutenproject/GlutenConfig.scala | 1733 --------------
.../datasource/GlutenFormatWriterInjects.scala | 94 -
.../datasource/GlutenOrcWriterInjects.scala | 32 -
.../datasource/GlutenParquetWriterInjects.scala | 47 -
.../glutenproject/expression/ExpressionNames.scala | 297 ---
.../scala/io/glutenproject/expression/Sig.scala | 31 -
.../glutenproject/metrics/GlutenTimeMetric.scala | 47 -
.../glutenproject/sql/shims/SparkShimLoader.scala | 79 -
.../sql/shims/SparkShimProvider.scala | 30 -
.../io/glutenproject/sql/shims/SparkShims.scala | 198 --
.../main/scala/io/glutenproject/utils/Arm.scala | 30 -
.../scala/org/apache/gluten/GlutenConfig.scala | 1733 ++++++++++++++
.../datasource/GlutenFormatWriterInjects.scala | 94 +
.../datasource/GlutenOrcWriterInjects.scala | 32 +
.../datasource/GlutenParquetWriterInjects.scala | 47 +
.../apache/gluten/expression/ExpressionNames.scala | 297 +++
.../scala/org/apache/gluten/expression/Sig.scala | 31 +
.../apache/gluten/metrics/GlutenTimeMetric.scala | 47 +
.../apache/gluten/sql/shims/SparkShimLoader.scala | 79 +
.../gluten/sql/shims/SparkShimProvider.scala | 30 +
.../org/apache/gluten/sql/shims/SparkShims.scala | 198 ++
.../main/scala/org/apache/gluten/utils/Arm.scala | 30 +
shims/pom.xml | 2 +-
shims/spark32/pom.xml | 4 +-
.../io.glutenproject.sql.shims.SparkShimProvider | 1 -
.../org.apache.gluten.sql.shims.SparkShimProvider | 1 +
.../execution/GenerateTreeStringShim.scala | 64 -
.../sql/shims/spark32/Spark32Shims.scala | 205 --
.../sql/shims/spark32/SparkShimProvider.scala | 41 -
.../gluten/execution/GenerateTreeStringShim.scala | 64 +
.../gluten/sql/shims/spark32/Spark32Shims.scala | 205 ++
.../sql/shims/spark32/SparkShimProvider.scala | 41 +
.../sql/execution/FileSourceScanExecShim.scala | 2 +-
.../datasources/FileFormatDataWriter.scala | 2 +-
.../execution/datasources/FileFormatWriter.scala | 4 +-
.../execution/datasources/orc/OrcFileFormat.scala | 4 +-
.../datasources/parquet/ParquetFileFormat.scala | 4 +-
.../spark/sql/hive/execution/HiveFileFormat.scala | 4 +-
.../sql/shims/spark32/Spark32ShimsSuite.scala | 35 -
.../sql/shims/spark32/Spark32ShimsSuite.scala | 35 +
shims/spark33/pom.xml | 4 +-
.../io.glutenproject.sql.shims.SparkShimProvider | 1 -
.../org.apache.gluten.sql.shims.SparkShimProvider | 1 +
.../execution/GenerateTreeStringShim.scala | 64 -
.../sql/shims/spark33/Spark33Shims.scala | 279 ---
.../sql/shims/spark33/SparkShimProvider.scala | 41 -
.../gluten/execution/GenerateTreeStringShim.scala | 64 +
.../gluten/sql/shims/spark33/Spark33Shims.scala | 279 +++
.../sql/shims/spark33/SparkShimProvider.scala | 41 +
.../sql/execution/FileSourceScanExecShim.scala | 2 +-
.../datasources/FileFormatDataWriter.scala | 2 +-
.../execution/datasources/FileFormatWriter.scala | 4 +-
.../execution/datasources/orc/OrcFileFormat.scala | 4 +-
.../datasources/parquet/ParquetFileFormat.scala | 4 +-
.../spark/sql/hive/execution/HiveFileFormat.scala | 4 +-
shims/spark34/pom.xml | 4 +-
.../io.glutenproject.sql.shims.SparkShimProvider | 1 -
.../org.apache.gluten.sql.shims.SparkShimProvider | 1 +
.../execution/GenerateTreeStringShim.scala | 64 -
.../sql/shims/spark34/Spark34Shims.scala | 357 ---
.../sql/shims/spark34/SparkShimProvider.scala | 41 -
.../gluten/execution/GenerateTreeStringShim.scala | 64 +
.../gluten/sql/shims/spark34/Spark34Shims.scala | 357 +++
.../sql/shims/spark34/SparkShimProvider.scala | 41 +
.../sql/execution/FileSourceScanExecShim.scala | 2 +-
shims/spark35/pom.xml | 4 +-
.../io.glutenproject.sql.shims.SparkShimProvider | 1 -
.../org.apache.gluten.sql.shims.SparkShimProvider | 1 +
.../execution/GenerateTreeStringShim.scala | 64 -
.../sql/shims/spark35/Spark35Shims.scala | 322 ---
.../sql/shims/spark35/SparkShimProvider.scala | 41 -
.../gluten/execution/GenerateTreeStringShim.scala | 64 +
.../gluten/sql/shims/spark35/Spark35Shims.scala | 322 +++
.../sql/shims/spark35/SparkShimProvider.scala | 41 +
.../sql/execution/FileSourceScanExecShim.scala | 2 +-
tools/gluten-it/common/pom.xml | 8 +-
.../java/io/glutenproject/integration/tpc/Tpc.java | 40 -
.../io/glutenproject/integration/tpc/TpcMixin.java | 168 --
.../integration/tpc/command/DataGenMixin.java | 42 -
.../integration/tpc/command/DataGenOnly.java | 38 -
.../integration/tpc/command/Parameterized.java | 128 -
.../integration/tpc/command/Queries.java | 56 -
.../integration/tpc/command/QueriesCompare.java | 53 -
.../integration/tpc/command/SparkRunModes.java | 152 --
.../integration/tpc/command/SparkShell.java | 41 -
.../org/apache/gluten/integration/tpc/Tpc.java | 40 +
.../apache/gluten/integration/tpc/TpcMixin.java | 168 ++
.../integration/tpc/command/DataGenMixin.java | 42 +
.../integration/tpc/command/DataGenOnly.java | 38 +
.../integration/tpc/command/Parameterized.java | 128 +
.../gluten/integration/tpc/command/Queries.java | 56 +
.../integration/tpc/command/QueriesCompare.java | 53 +
.../integration/tpc/command/SparkRunModes.java | 152 ++
.../gluten/integration/tpc/command/SparkShell.java | 41 +
.../glutenproject/integration/stat/RamStat.scala | 89 -
.../glutenproject/integration/tpc/Constants.scala | 110 -
.../io/glutenproject/integration/tpc/DataGen.scala | 68 -
.../glutenproject/integration/tpc/ShimUtils.scala | 37 -
.../glutenproject/integration/tpc/TpcRunner.scala | 70 -
.../glutenproject/integration/tpc/TpcSuite.scala | 214 --
.../integration/tpc/action/Actions.scala | 26 -
.../integration/tpc/action/DataGenOnly.scala | 30 -
.../integration/tpc/action/Parameterized.scala | 294 ---
.../integration/tpc/action/Queries.scala | 187 --
.../integration/tpc/action/QueriesCompare.scala | 227 --
.../integration/tpc/action/SparkShell.scala | 33 -
.../integration/tpc/ds/TpcdsDataGen.scala | 703 ------
.../integration/tpc/ds/TpcdsSuite.scala | 197 --
.../integration/tpc/h/TpchDataGen.scala | 357 ---
.../integration/tpc/h/TpchSuite.scala | 114 -
.../apache/gluten/integration/stat/RamStat.scala | 89 +
.../apache/gluten/integration/tpc/Constants.scala | 110 +
.../apache/gluten/integration/tpc/DataGen.scala | 68 +
.../apache/gluten/integration/tpc/ShimUtils.scala | 37 +
.../apache/gluten/integration/tpc/TpcRunner.scala | 70 +
.../apache/gluten/integration/tpc/TpcSuite.scala | 214 ++
.../gluten/integration/tpc/action/Actions.scala | 26 +
.../integration/tpc/action/DataGenOnly.scala | 30 +
.../integration/tpc/action/Parameterized.scala | 294 +++
.../gluten/integration/tpc/action/Queries.scala | 187 ++
.../integration/tpc/action/QueriesCompare.scala | 227 ++
.../gluten/integration/tpc/action/SparkShell.scala | 33 +
.../gluten/integration/tpc/ds/TpcdsDataGen.scala | 703 ++++++
.../gluten/integration/tpc/ds/TpcdsSuite.scala | 197 ++
.../gluten/integration/tpc/h/TpchDataGen.scala | 357 +++
.../gluten/integration/tpc/h/TpchSuite.scala | 114 +
.../io/glutenproject/integration/tpc/TpcTest.java | 23 -
.../org/apache/gluten/integration/tpc/TpcTest.java | 23 +
tools/gluten-it/package/pom.xml | 6 +-
tools/gluten-it/pom.xml | 6 +-
tools/gluten-it/sbin/gluten-it.sh | 2 +-
tools/gluten-te/ubuntu/tpc.sh | 2 +-
tools/workload/tpcds/run_tpcds/run_tpcds.sh | 2 +-
.../dwrf_dataset/tpch_convert_parquet_dwrf.sh | 2 +-
tools/workload/tpch/run_tpch/tpch_parquet.sh | 2 +-
1533 files changed, 108339 insertions(+), 108321 deletions(-)
diff --git a/.github/workflows/velox_be.yml.deprecated b/.github/workflows/velox_be.yml.deprecated
index 85db767b9..d095af64d 100644
--- a/.github/workflows/velox_be.yml.deprecated
+++ b/.github/workflows/velox_be.yml.deprecated
@@ -84,8 +84,8 @@ jobs:
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
+ mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
# Cpp micro benchmarks will use generated files from unit test in backends-velox module.
- name: Run micro benchmarks
run: |
@@ -221,8 +221,8 @@ jobs:
- name: Build and Run unit test for Spark 3.3.1 (other tests)
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
+ mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
- name: Copy golden files from container to host
if: failure()
run: |
@@ -304,8 +304,8 @@ jobs:
- name: Build and Run unit test for Spark 3.4.2 (other tests)
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
+ mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
- name: Copy golden files from container to host
if: failure()
run: |
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
index c54a6a1e1..124efc53a 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -393,8 +393,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn -ntp test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn -ntp test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
@@ -480,8 +480,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn -ntp test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn -ntp test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
@@ -568,8 +568,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
- mvn -ntp test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
+ mvn -ntp test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
diff --git a/.scalafmt.conf b/.scalafmt.conf
index ce11260c4..e65c0217f 100644
--- a/.scalafmt.conf
+++ b/.scalafmt.conf
@@ -47,7 +47,7 @@ rewrite.rules = [AvoidInfix, Imports, RedundantBraces, SortModifiers]
# Imports
rewrite.imports.sort = scalastyle
rewrite.imports.groups = [
- ["io.glutenproject\\..*"],
+ ["org.apache.gluten\\..*"],
["io.substrait.spark\\..*"],
["org.apache.spark\\..*"],
[".*"],
diff --git a/README.md b/README.md
index 76ce5fbe3..3de89b0a7 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,7 @@ One way is to use released jar. Here is a simple example. Currently, only centos
```
spark-shell \
--master yarn --deploy-mode client \
- --conf spark.plugins=io.glutenproject.GlutenPlugin \
+ --conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=20g \
--conf spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager \
@@ -85,7 +85,7 @@ Alternatively, you can build gluten from source, then do some configurations to
export gluten_jar = /PATH/TO/GLUTEN/backends-velox/target/<gluten-jar>
spark-shell
--master yarn --deploy-mode client \
- --conf spark.plugins=io.glutenproject.GlutenPlugin \
+ --conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=20g \
--conf spark.driver.extraClassPath=${gluten_jar} \
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index dd4240a3a..76e184f38 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -4,7 +4,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>gluten-parent</artifactId>
- <groupId>io.glutenproject</groupId>
+ <groupId>org.apache.gluten</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -15,7 +15,7 @@
<dependencies>
<dependency>
- <groupId>io.glutenproject</groupId>
+ <groupId>org.apache.gluten</groupId>
<artifactId>gluten-core</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
@@ -32,7 +32,7 @@
<version>${custom.protobuf.version}</version>
</dependency>
<dependency>
- <groupId>io.glutenproject</groupId>
+ <groupId>org.apache.gluten</groupId>
<artifactId>gluten-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
index 2042dad39..498b7ff4f 100644
--- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
+++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.delta.commands
-import io.glutenproject.expression.ConverterUtils
+import org.apache.gluten.expression.ConverterUtils
import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException}
import org.apache.spark.internal.Logging
diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
index 2042dad39..498b7ff4f 100644
--- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
+++ b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.delta.commands
-import io.glutenproject.expression.ConverterUtils
+import org.apache.gluten.expression.ConverterUtils
import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException}
import org.apache.spark.internal.Logging
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/execution/ColumnarNativeIterator.java b/backends-clickhouse/src/main/java/io/glutenproject/execution/ColumnarNativeIterator.java
deleted file mode 100644
index a069d2df4..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/execution/ColumnarNativeIterator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.execution;
-
-import io.glutenproject.vectorized.CHColumnVector;
-import io.glutenproject.vectorized.GeneralInIterator;
-
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-import java.util.Iterator;
-
-public class ColumnarNativeIterator extends GeneralInIterator implements Iterator<byte[]> {
-
- public ColumnarNativeIterator(Iterator<ColumnarBatch> delegated) {
- super(delegated);
- }
-
- private transient ColumnarBatch nextBatch = null;
-
- private static byte[] longtoBytes(long data) {
- return new byte[] {
- (byte) ((data >> 56) & 0xff),
- (byte) ((data >> 48) & 0xff),
- (byte) ((data >> 40) & 0xff),
- (byte) ((data >> 32) & 0xff),
- (byte) ((data >> 24) & 0xff),
- (byte) ((data >> 16) & 0xff),
- (byte) ((data >> 8) & 0xff),
- (byte) ((data >> 0) & 0xff),
- };
- }
-
- @Override
- public boolean hasNext() {
- while (delegated.hasNext()) {
- nextBatch = delegated.next();
- if (nextBatch.numRows() > 0) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public ColumnarBatch nextColumnarBatch() {
- return nextBatch;
- }
-
- @Override
- public byte[] next() {
- ColumnarBatch nextBatch = nextColumnarBatch();
- if (nextBatch.numRows() > 0) {
- CHColumnVector col = (CHColumnVector) nextBatch.column(0);
- return longtoBytes(col.getBlockAddress());
- } else {
- throw new IllegalStateException();
- }
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/execution/SparkRowIterator.java b/backends-clickhouse/src/main/java/io/glutenproject/execution/SparkRowIterator.java
deleted file mode 100644
index 564a3cbd8..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/execution/SparkRowIterator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.execution;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.Iterator;
-
-public class SparkRowIterator implements Iterator<byte[]> {
- private final scala.collection.Iterator<byte[]> delegated;
- private final int defaultBufSize = 4096;
- private byte[] lastRowBuf;
-
- public SparkRowIterator(scala.collection.Iterator<byte[]> delegated) {
- this.delegated = delegated;
- lastRowBuf = null;
- }
-
- @Override
- public boolean hasNext() {
- return lastRowBuf != null || delegated.hasNext();
- }
-
- @Override
- public byte[] next() {
- return delegated.next();
- }
-
- protected ByteBuffer createByteBuffer(int bufSize) {
- ByteBuffer buf;
- // 8: one length and one end flag
- if (bufSize + 8 > defaultBufSize) {
- buf = ByteBuffer.allocateDirect(bufSize + 8);
- } else {
- buf = ByteBuffer.allocateDirect(defaultBufSize);
- }
- buf.order(ByteOrder.LITTLE_ENDIAN);
- return buf;
- }
-
- public ByteBuffer nextBatch() {
- ByteBuffer buf = null;
- if (lastRowBuf != null) {
- buf = createByteBuffer(lastRowBuf.length);
- buf.putInt(lastRowBuf.length);
- buf.put(lastRowBuf);
- // make the end flag
- buf.putInt(-1);
- lastRowBuf = null;
- return buf;
- } else {
- while (delegated.hasNext()) {
- lastRowBuf = delegated.next();
- if (buf == null) {
- buf = createByteBuffer(lastRowBuf.length);
- }
- if (buf.remaining() < lastRowBuf.length + 8) {
- break;
- } else {
- buf.putInt(lastRowBuf.length);
- buf.put(lastRowBuf);
- lastRowBuf = null;
- }
- }
- buf.putInt(-1);
- return buf;
- }
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHManagedCHReservationListener.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHManagedCHReservationListener.java
deleted file mode 100644
index dfb5ab768..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHManagedCHReservationListener.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-import io.glutenproject.GlutenConfig;
-import io.glutenproject.memory.SimpleMemoryUsageRecorder;
-import io.glutenproject.memory.memtarget.MemoryTarget;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class CHManagedCHReservationListener implements CHReservationListener {
-
- private static final Logger LOG = LoggerFactory.getLogger(CHManagedCHReservationListener.class);
-
- private MemoryTarget target;
- private final SimpleMemoryUsageRecorder usage;
- private final boolean throwIfMemoryExceed =
- GlutenConfig.getConf().chColumnarThrowIfMemoryExceed();
- private volatile boolean open = true;
-
- private final AtomicLong currentMemory = new AtomicLong(0L);
-
- public CHManagedCHReservationListener(MemoryTarget target, SimpleMemoryUsageRecorder usage) {
- this.target = target;
- this.usage = usage;
- }
-
- @Override
- public void reserveOrThrow(long size) {
- if (!throwIfMemoryExceed) {
- reserve(size);
- return;
- }
-
- synchronized (this) {
- if (!open) {
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("reserve memory size from native: %d", size));
- }
- long granted = target.borrow(size);
- if (granted < size) {
- target.repay(granted);
- throw new UnsupportedOperationException(
- "Not enough spark off-heap execution memory. "
- + "Acquired: "
- + size
- + ", granted: "
- + granted
- + ". "
- + "Try tweaking config option spark.memory.offHeap.size to "
- + "get larger space to run this application. ");
- }
- currentMemory.addAndGet(size);
- usage.inc(size);
- }
- }
-
- @Override
- public long reserve(long size) {
- synchronized (this) {
- if (!open) {
- return 0L;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("reserve memory (without exception) size from native: %d", size));
- }
- long granted = target.borrow(size);
- if (granted < size && (LOG.isWarnEnabled())) {
- LOG.warn(
- String.format(
- "Not enough spark off-heap execution memory. "
- + "Acquired: %d, granted: %d. Try tweaking config option "
- + "spark.memory.offHeap.size to get larger space "
- + "to run this application.",
- size, granted));
- }
- currentMemory.addAndGet(granted);
- usage.inc(size);
- return granted;
- }
- }
-
- @Override
- public long unreserve(long size) {
- synchronized (this) {
- if (!open) {
- return 0L;
- }
- long memoryToFree = size;
- if ((currentMemory.get() - size) < 0L) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- String.format(
- "The current used memory' %d will be less than 0(%d) after free %d",
- currentMemory.get(), currentMemory.get() - size, size));
- }
- memoryToFree = currentMemory.get();
- }
- if (memoryToFree == 0L) {
- return memoryToFree;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("unreserve memory size from native: %d", memoryToFree));
- }
- target.repay(memoryToFree);
- currentMemory.addAndGet(-memoryToFree);
- usage.inc(-size);
- return memoryToFree;
- }
- }
-
- @Override
- public void inactivate() {
- synchronized (this) {
- // for some reasons, memory audit in the native code may not be 100% accurate
- // we'll allow the inaccuracy
- if (currentMemory.get() > 0) {
- unreserve(currentMemory.get());
- } else if (currentMemory.get() < 0) {
- reserve(currentMemory.get());
- }
- currentMemory.set(0L);
-
- target = null; // make it gc reachable
- open = false;
- }
- }
-
- @Override
- public long currentMemory() {
- return currentMemory.get();
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocator.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocator.java
deleted file mode 100644
index 44163d95c..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-/**
- * Like {@link io.glutenproject.vectorized.NativePlanEvaluator}, this along with {@link
- * CHNativeMemoryAllocators}, as built-in toolkit for managing native memory allocations.
- */
-public class CHNativeMemoryAllocator {
-
- private final long nativeInstanceId;
- private final CHReservationListener listener;
-
- public CHNativeMemoryAllocator(long nativeInstanceId, CHReservationListener listener) {
- this.nativeInstanceId = nativeInstanceId;
- this.listener = listener;
- }
-
- public static CHNativeMemoryAllocator getDefault() {
- return new CHNativeMemoryAllocator(getDefaultAllocator(), CHReservationListener.NOOP);
- }
-
- public static CHNativeMemoryAllocator getDefaultForUT() {
- return new CHNativeMemoryAllocator(
- createListenableAllocator(CHReservationListener.NOOP), CHReservationListener.NOOP);
- }
-
- public static CHNativeMemoryAllocator createListenable(CHReservationListener listener) {
- return new CHNativeMemoryAllocator(createListenableAllocator(listener), listener);
- }
-
- public CHReservationListener listener() {
- return listener;
- }
-
- public long getNativeInstanceId() {
- return this.nativeInstanceId;
- }
-
- public long getBytesAllocated() {
- if (this.nativeInstanceId == -1L) return 0;
- return bytesAllocated(this.nativeInstanceId);
- }
-
- public void close() {
- if (this.nativeInstanceId == -1L) return;
- releaseAllocator(this.nativeInstanceId);
- }
-
- private static native long getDefaultAllocator();
-
- private static native long createListenableAllocator(CHReservationListener listener);
-
- private static native void releaseAllocator(long allocatorId);
-
- private static native long bytesAllocated(long allocatorId);
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManager.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManager.java
deleted file mode 100644
index 6940840eb..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManager.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-import org.apache.spark.util.TaskResource;
-
-/** Resource manager implementation that manages a {@link CHNativeMemoryAllocator}. */
-public interface CHNativeMemoryAllocatorManager extends TaskResource {
- CHNativeMemoryAllocator getManaged();
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java
deleted file mode 100644
index 3666c58b5..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-public class CHNativeMemoryAllocatorManagerImpl implements CHNativeMemoryAllocatorManager {
- private final CHNativeMemoryAllocator managed;
-
- public CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator managed) {
- this.managed = managed;
- }
-
- @Override
- public void release() {
- managed.close();
- managed.listener().inactivate();
- }
-
- @Override
- public CHNativeMemoryAllocator getManaged() {
- return managed;
- }
-
- @Override
- public int priority() {
- return 0; // lowest priority
- }
-
- @Override
- public String resourceName() {
- return "CHNativeMemoryAllocatorManager_" + managed.getNativeInstanceId();
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocators.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocators.java
deleted file mode 100644
index 2c6db5913..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHNativeMemoryAllocators.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-import io.glutenproject.memory.SimpleMemoryUsageRecorder;
-import io.glutenproject.memory.memtarget.MemoryTargets;
-import io.glutenproject.memory.memtarget.Spiller;
-
-import org.apache.spark.memory.TaskMemoryManager;
-import org.apache.spark.util.TaskResources;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Built-in toolkit for managing native memory allocations. To use the facility, one should import
- * Gluten's C++ library then create the c++ instance using following example code:
- *
- * <p>```c++ auto* allocator = reinterpret_cast<gluten::memory::MemoryAllocator*>(allocator_id); ```
- *
- * <p>The ID "allocator_id" can be retrieved from Java API {@link
- * CHNativeMemoryAllocator#getNativeInstanceId()}.
- *
- * <p>FIXME: to export the native APIs in a standard way
- */
-public abstract class CHNativeMemoryAllocators {
- private CHNativeMemoryAllocators() {}
-
- private static final CHNativeMemoryAllocator GLOBAL = CHNativeMemoryAllocator.getDefault();
-
- private static CHNativeMemoryAllocatorManager createNativeMemoryAllocatorManager(
- String name,
- TaskMemoryManager taskMemoryManager,
- List<Spiller> spillers,
- SimpleMemoryUsageRecorder usage) {
-
- CHManagedCHReservationListener rl =
- new CHManagedCHReservationListener(
- MemoryTargets.newConsumer(taskMemoryManager, name, spillers, Collections.emptyMap()),
- usage);
- return new CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator.createListenable(rl));
- }
-
- public static CHNativeMemoryAllocator contextInstance() {
- if (!TaskResources.inSparkTask()) {
- return globalInstance();
- }
-
- final String id = CHNativeMemoryAllocatorManager.class.toString();
- if (!TaskResources.isResourceRegistered(id)) {
- final CHNativeMemoryAllocatorManager manager =
- createNativeMemoryAllocatorManager(
- "ContextInstance",
- TaskResources.getLocalTaskContext().taskMemoryManager(),
- Collections.emptyList(),
- TaskResources.getSharedUsage());
- TaskResources.addResource(id, manager);
- }
- return ((CHNativeMemoryAllocatorManager) TaskResources.getResource(id)).getManaged();
- }
-
- public static CHNativeMemoryAllocator contextInstanceForUT() {
- return CHNativeMemoryAllocator.getDefaultForUT();
- }
-
- public static CHNativeMemoryAllocator createSpillable(String name, Spiller... spillers) {
- if (!TaskResources.inSparkTask()) {
- throw new IllegalStateException("spiller must be used in a Spark task");
- }
-
- final CHNativeMemoryAllocatorManager manager =
- createNativeMemoryAllocatorManager(
- name,
- TaskResources.getLocalTaskContext().taskMemoryManager(),
- Arrays.asList(spillers),
- TaskResources.getSharedUsage());
- TaskResources.addAnonymousResource(manager);
- // force add memory consumer to task memory manager, will release by inactivate
- manager.getManaged().listener().reserve(1);
- return manager.getManaged();
- }
-
- public static CHNativeMemoryAllocator globalInstance() {
- return GLOBAL;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHReservationListener.java b/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHReservationListener.java
deleted file mode 100644
index 8328543d7..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/memory/alloc/CHReservationListener.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.memory.alloc;
-
-public interface CHReservationListener {
- CHReservationListener NOOP =
- new CHReservationListener() {
- @Override
- public void reserveOrThrow(long size) {}
-
- @Override
- public long reserve(long size) {
- return 0L;
- }
-
- @Override
- public long unreserve(long size) {
- return 0L;
- }
-
- @Override
- public void inactivate() {}
-
- @Override
- public long currentMemory() {
- return 0L;
- }
- };
-
- long reserve(long size);
-
- void reserveOrThrow(long size);
-
- long unreserve(long size);
-
- void inactivate();
-
- long currentMemory();
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsData.java b/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsData.java
deleted file mode 100644
index 40c469ec3..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsData.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.metrics;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.List;
-
-public class MetricsData {
-
- protected long id;
- protected String name;
- protected long time;
-
- @JsonProperty("input_wait_time")
- protected long inputWaitTime;
-
- @JsonProperty("output_wait_time")
- protected long outputWaitTime;
-
- protected long inputRows = 0;
- protected long inputVectors = 0;
- protected long inputBytes = 0;
- protected long outputRows = 0;
- protected long outputVectors = 0;
- protected long outputBytes = 0;
- protected List<MetricsStep> steps;
-
- public long getId() {
- return id;
- }
-
- public void setId(long id) {
- this.id = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getTime() {
- return time;
- }
-
- public void setTime(long time) {
- this.time = time;
- }
-
- public List<MetricsStep> getSteps() {
- return steps;
- }
-
- public void setSteps(List<MetricsStep> steps) {
- this.steps = steps;
- }
-
- public long getInputRows() {
- return inputRows;
- }
-
- public void setInputRows(long inputRows) {
- this.inputRows = inputRows;
- }
-
- public long getInputVectors() {
- return inputVectors;
- }
-
- public void setInputVectors(long inputVectors) {
- this.inputVectors = inputVectors;
- }
-
- public long getInputBytes() {
- return inputBytes;
- }
-
- public void setInputBytes(long inputBytes) {
- this.inputBytes = inputBytes;
- }
-
- public long getOutputRows() {
- return outputRows;
- }
-
- public void setOutputRows(long outputRows) {
- this.outputRows = outputRows;
- }
-
- public long getOutputVectors() {
- return outputVectors;
- }
-
- public void setOutputVectors(long outputVectors) {
- this.outputVectors = outputVectors;
- }
-
- public long getOutputBytes() {
- return outputBytes;
- }
-
- public void setOutputBytes(long outputBytes) {
- this.outputBytes = outputBytes;
- }
-
- public long getInputWaitTime() {
- return inputWaitTime;
- }
-
- public void setInputWaitTime(long inputWaitTime) {
- this.inputWaitTime = inputWaitTime;
- }
-
- public long getOutputWaitTime() {
- return outputWaitTime;
- }
-
- public void setOutputWaitTime(long outputWaitTime) {
- this.outputWaitTime = outputWaitTime;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsProcessor.java b/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsProcessor.java
deleted file mode 100644
index ff2504d20..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsProcessor.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.metrics;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class MetricsProcessor {
-
- protected String name;
- protected long time;
-
- @JsonProperty("input_rows")
- protected long inputRows = 0;
-
- protected long inputVectors = 0;
-
- @JsonProperty("input_bytes")
- protected long inputBytes = 0;
-
- @JsonProperty("output_rows")
- protected long outputRows = 0;
-
- protected long outputVectors = 0;
-
- @JsonProperty("output_bytes")
- protected long outputBytes = 0;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getTime() {
- return time;
- }
-
- public void setTime(long time) {
- this.time = time;
- }
-
- public long getInputRows() {
- return inputRows;
- }
-
- public void setInputRows(long inputRows) {
- this.inputRows = inputRows;
- }
-
- public long getInputVectors() {
- return inputVectors;
- }
-
- public void setInputVectors(long inputVectors) {
- this.inputVectors = inputVectors;
- }
-
- public long getInputBytes() {
- return inputBytes;
- }
-
- public void setInputBytes(long inputBytes) {
- this.inputBytes = inputBytes;
- }
-
- public long getOutputRows() {
- return outputRows;
- }
-
- public void setOutputRows(long outputRows) {
- this.outputRows = outputRows;
- }
-
- public long getOutputVectors() {
- return outputVectors;
- }
-
- public void setOutputVectors(long outputVectors) {
- this.outputVectors = outputVectors;
- }
-
- public long getOutputBytes() {
- return outputBytes;
- }
-
- public void setOutputBytes(long outputBytes) {
- this.outputBytes = outputBytes;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsStep.java b/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsStep.java
deleted file mode 100644
index d1714d825..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/metrics/MetricsStep.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.metrics;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.List;
-
-public class MetricsStep {
-
- protected String name;
- protected String description;
- protected List<MetricsProcessor> processors;
-
- @JsonProperty("total_marks_pk")
- protected long totalMarksPk;
-
- @JsonProperty("selected_marks_pk")
- protected long selectedMarksPk;
-
- @JsonProperty("selected_marks")
- protected long selectedMarks;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public List<MetricsProcessor> getProcessors() {
- return processors;
- }
-
- public void setProcessors(List<MetricsProcessor> processors) {
- this.processors = processors;
- }
-
- public void setTotalMarksPk(long totalMarksPk) {
- this.totalMarksPk = totalMarksPk;
- }
-
- public void setSelectedMarksPk(long selectedMarksPk) {
- this.selectedMarksPk = selectedMarksPk;
- }
-
- public long getSelectedMarks() {
- return selectedMarks;
- }
-
- public void setSelectedMarks(long selectedMarks) {
- this.selectedMarks = selectedMarks;
- }
-
- public long getTotalMarksPk() {
- return totalMarksPk;
- }
-
- public long getSelectedMarksPk() {
- return selectedMarksPk;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/metrics/NativeMetrics.java b/backends-clickhouse/src/main/java/io/glutenproject/metrics/NativeMetrics.java
deleted file mode 100644
index 0d9160c97..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/metrics/NativeMetrics.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.metrics;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.collections.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class NativeMetrics implements IMetrics {
-
- private static final Logger LOG = LoggerFactory.getLogger(NativeMetrics.class);
-
- public List<MetricsData> metricsDataList;
- public String metricsJson;
-
- public NativeMetrics(String metricsJson) {
- this.metricsJson = metricsJson;
- this.metricsDataList = NativeMetrics.deserializeMetricsJson(this.metricsJson);
- }
-
- public void setFinalOutputMetrics(long outputRowCount, long outputVectorCount) {
- if (CollectionUtils.isNotEmpty(this.metricsDataList)) {
- int listSize = this.metricsDataList.size();
- this.metricsDataList.get(listSize - 1).outputVectors = outputVectorCount;
- this.metricsDataList.get(listSize - 1).outputRows = outputRowCount;
-
- // skip 'kFetch' and 'kSort'
- if (listSize > 2
- && this.metricsDataList.get(listSize - 1).name.equals("kFetch")
- && this.metricsDataList.get(listSize - 2).name.equals("kSort")) {
- this.metricsDataList.get(listSize - 3).outputVectors = outputVectorCount;
- this.metricsDataList.get(listSize - 3).outputRows = outputRowCount;
- }
- }
- }
-
- /** Deserialize metrics json string to MetricsData */
- public static List<MetricsData> deserializeMetricsJson(String metricsJson) {
- if (metricsJson != null && !metricsJson.isEmpty()) {
- ObjectMapper mapper = new ObjectMapper();
- try {
- List<MetricsData> metricsDataList =
- mapper.readValue(metricsJson, new TypeReference<List<MetricsData>>() {});
- Collections.reverse(metricsDataList);
- return metricsDataList;
- } catch (Exception e) {
- LOG.error("Deserialize metrics json string error:", e);
- return new ArrayList<MetricsData>();
- }
- }
- return new ArrayList<MetricsData>();
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/metrics/OperatorMetrics.java b/backends-clickhouse/src/main/java/io/glutenproject/metrics/OperatorMetrics.java
deleted file mode 100644
index fd4018138..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/metrics/OperatorMetrics.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.metrics;
-
-import io.glutenproject.substrait.AggregationParams;
-import io.glutenproject.substrait.JoinParams;
-
-import java.util.List;
-
-public class OperatorMetrics implements IOperatorMetrics {
-
- public List<MetricsData> metricsList;
- public JoinParams joinParams;
- public AggregationParams aggParams;
-
- /** Create an instance for operator metrics. */
- public OperatorMetrics(
- List<MetricsData> metricsList, JoinParams joinParams, AggregationParams aggParams) {
- this.metricsList = metricsList;
- this.aggParams = aggParams;
- this.joinParams = joinParams;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/utils/SnowflakeIdWorker.java b/backends-clickhouse/src/main/java/io/glutenproject/utils/SnowflakeIdWorker.java
deleted file mode 100644
index e049e611e..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/utils/SnowflakeIdWorker.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.utils;
-
-import io.glutenproject.exception.GlutenException;
-
-import org.apache.spark.SparkEnv;
-import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig;
-
-/**
- * An object that generates IDs. This is broken into a separate class in case we ever want to
- * support multiple worker threads per process. Refer to twitter-archive Snowflake.
- */
-public class SnowflakeIdWorker {
-
- // ==============================Singleton=====================================
- private static volatile SnowflakeIdWorker INSTANCE;
- // ==============================Fields===========================================
- private final long twepoch = 1640966400L;
- private final long workerIdBits = 6L;
- private final long maxWorkerId = -1L ^ (-1L << workerIdBits);
- private final long sequenceBits = 16L;
- private final long workerIdShift = sequenceBits;
- private final long timestampLeftShift = sequenceBits + workerIdBits;
- private final long sequenceMask = -1L ^ (-1L << sequenceBits);
- private long workerId;
- private long sequence = 0L;
- private long lastTimestamp = -1L;
-
- public SnowflakeIdWorker(long workerId) {
- if (workerId > maxWorkerId || workerId < 0) {
- throw new IllegalArgumentException(
- String.format("worker Id can't be greater than %d or less than 0", maxWorkerId));
- }
- this.workerId = workerId;
- }
-
- // ==============================Constructors=====================================
-
- public static SnowflakeIdWorker getInstance() {
- if (INSTANCE == null) {
- synchronized (SnowflakeIdWorker.class) {
- if (INSTANCE == null) {
- if (!SparkEnv.get().conf().contains(ClickHouseConfig.CLICKHOUSE_WORKER_ID())) {
- throw new IllegalArgumentException(
- "Please set an unique value to " + ClickHouseConfig.CLICKHOUSE_WORKER_ID());
- }
- INSTANCE =
- new SnowflakeIdWorker(
- SparkEnv.get().conf().getLong(ClickHouseConfig.CLICKHOUSE_WORKER_ID(), 0));
- }
- }
- }
- return INSTANCE;
- }
-
- // ==============================Methods==========================================
- public synchronized long nextId() {
- long timestamp = timeGen();
-
- if (timestamp < lastTimestamp) {
- throw new GlutenException(
- String.format(
- "Clock moved backwards. Refusing to generate id for %d milliseconds",
- lastTimestamp - timestamp));
- }
-
- if (lastTimestamp == timestamp) {
- sequence = (sequence + 1) & sequenceMask;
- if (sequence == 0) {
- timestamp = tilNextMillis(lastTimestamp);
- }
- } else {
- sequence = 0L;
- }
-
- lastTimestamp = timestamp;
-
- return ((timestamp - twepoch) << timestampLeftShift) //
- | (workerId << workerIdShift) //
- | sequence;
- }
-
- protected long tilNextMillis(long lastTimestamp) {
- long timestamp = timeGen();
- while (timestamp <= lastTimestamp) {
- timestamp = timeGen();
- }
- return timestamp;
- }
-
- protected long timeGen() {
- return System.currentTimeMillis() / 1000L;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BatchIterator.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BatchIterator.java
deleted file mode 100644
index f87948368..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BatchIterator.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.metrics.IMetrics;
-
-import org.apache.spark.sql.execution.utils.CHExecUtil;
-import org.apache.spark.sql.vectorized.ColumnVector;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-import java.io.IOException;
-
-public class BatchIterator extends GeneralOutIterator {
- private final long handle;
-
- public BatchIterator(long handle) {
- super();
- this.handle = handle;
- }
-
- @Override
- public String getId() {
- // Using native handle as identifier
- return String.valueOf(handle);
- }
-
- private native boolean nativeHasNext(long nativeHandle);
-
- private native byte[] nativeNext(long nativeHandle);
-
- private native long nativeCHNext(long nativeHandle);
-
- private native void nativeClose(long nativeHandle);
-
- private native IMetrics nativeFetchMetrics(long nativeHandle);
-
- @Override
- public boolean hasNextInternal() throws IOException {
- return nativeHasNext(handle);
- }
-
- @Override
- public ColumnarBatch nextInternal() throws IOException {
- long block = nativeCHNext(handle);
- CHNativeBlock nativeBlock = new CHNativeBlock(block);
- int cols = nativeBlock.numColumns();
- ColumnVector[] columnVectors = new ColumnVector[cols];
- for (int i = 0; i < cols; i++) {
- columnVectors[i] =
- new CHColumnVector(
- CHExecUtil.inferSparkDataType(nativeBlock.getTypeByPosition(i)), block, i);
- }
- return new ColumnarBatch(columnVectors, nativeBlock.numRows());
- }
-
- @Override
- public IMetrics getMetricsInternal() throws IOException, ClassNotFoundException {
- return nativeFetchMetrics(handle);
- }
-
- @Override
- public void closeInternal() {
- nativeClose(handle);
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockOutputStream.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockOutputStream.java
deleted file mode 100644
index 9bdaa21ff..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockOutputStream.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import org.apache.spark.sql.execution.metric.SQLMetric;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-import org.apache.spark.storage.CHShuffleWriteStreamFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
-
-public class BlockOutputStream implements Closeable {
- private final long instance;
- private final OutputStream outputStream;
-
- private final SQLMetric dataSize;
-
- private boolean isClosed = false;
-
- public BlockOutputStream(
- OutputStream outputStream,
- byte[] buffer,
- SQLMetric dataSize,
- boolean compressionEnable,
- String defaultCompressionCodec,
- int bufferSize) {
- OutputStream unwrapOutputStream =
- CHShuffleWriteStreamFactory.unwrapSparkCompressionOutputStream(
- outputStream, compressionEnable);
- if (unwrapOutputStream != null) {
- this.outputStream = unwrapOutputStream;
- } else {
- this.outputStream = outputStream;
- compressionEnable = false;
- }
- this.instance =
- nativeCreate(
- this.outputStream, buffer, defaultCompressionCodec, compressionEnable, bufferSize);
- this.dataSize = dataSize;
- }
-
- private native long nativeCreate(
- OutputStream outputStream,
- byte[] buffer,
- String defaultCompressionCodec,
- boolean compressionEnable,
- int bufferSize);
-
- private native long nativeClose(long instance);
-
- private native void nativeWrite(long instance, long block);
-
- private native void nativeFlush(long instance);
-
- public void write(ColumnarBatch cb) {
- CHNativeBlock block = CHNativeBlock.fromColumnarBatch(cb);
- dataSize.add(block.totalBytes());
- nativeWrite(instance, block.blockAddress());
- }
-
- public void flush() throws IOException {
- nativeFlush(instance);
- this.outputStream.flush();
- }
-
- @Override
- public void close() throws IOException {
- if (!isClosed) {
- nativeClose(instance);
- this.outputStream.flush();
- this.outputStream.close();
- isClosed = true;
- }
- }
-
- @Override
- protected void finalize() throws Throwable {
- // FIXME: finalize
- close();
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockSplitIterator.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockSplitIterator.java
deleted file mode 100644
index 3c0d919f6..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/BlockSplitIterator.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-public class BlockSplitIterator implements Iterator<ColumnarBatch>, AutoCloseable {
-
- private long instance = 0;
-
- public BlockSplitIterator(Iterator<Long> in, IteratorOptions options) {
- this.instance =
- nativeCreate(
- new IteratorWrapper(in),
- options.getName(),
- options.getExpr(),
- options.getRequiredFields(),
- options.getPartitionNum(),
- options.getBufferSize(),
- options.getHashAlgorithm());
- }
-
- private native long nativeCreate(
- IteratorWrapper in,
- String name,
- String expr,
- String schema,
- int partitionNum,
- int bufferSize,
- String hashAlgorithm);
-
- private native void nativeClose(long instance);
-
- private native boolean nativeHasNext(long instance);
-
- @Override
- public boolean hasNext() {
- return nativeHasNext(instance);
- }
-
- private native long nativeNext(long instance);
-
- @Override
- public ColumnarBatch next() {
- CHNativeBlock block = new CHNativeBlock(nativeNext(instance));
- return block.toColumnarBatch();
- }
-
- private native int nativeNextPartitionId(long instance);
-
- public int nextPartitionId() {
- return nativeNextPartitionId(instance);
- }
-
- @Override
- public void close() throws Exception {
- nativeClose(instance);
- }
-
- public static class IteratorOptions implements Serializable {
- private static final long serialVersionUID = -1L;
- private int partitionNum;
- private String name;
- private int bufferSize;
- private String expr;
- private String requiredFields;
-
- private String hashAlgorithm;
-
- public int getPartitionNum() {
- return partitionNum;
- }
-
- public void setPartitionNum(int partitionNum) {
- this.partitionNum = partitionNum;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public void setBufferSize(int bufferSize) {
- this.bufferSize = bufferSize;
- }
-
- public String getExpr() {
- return expr;
- }
-
- public void setExpr(String expr) {
- this.expr = expr;
- }
-
- public String getRequiredFields() {
- return requiredFields;
- }
-
- public void setRequiredFields(String requiredFields) {
- this.requiredFields = requiredFields;
- }
-
- public String getHashAlgorithm() {
- return hashAlgorithm;
- }
-
- public void setHashAlgorithm(String hashAlgorithm) {
- this.hashAlgorithm = hashAlgorithm;
- }
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockConverterJniWrapper.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockConverterJniWrapper.java
deleted file mode 100644
index e4954c95f..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockConverterJniWrapper.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.execution.SparkRowIterator;
-import io.glutenproject.row.SparkRowInfo;
-
-public class CHBlockConverterJniWrapper {
-
- private CHBlockConverterJniWrapper() {
- // utility class
- }
-
- // for ch columnar -> spark row
- public static native SparkRowInfo convertColumnarToRow(long blockAddress, int[] masks);
-
- // for ch columnar -> spark row
- public static native void freeMemory(long address, long size);
-
- // for spark row -> ch columnar
- public static native long convertSparkRowsToCHColumn(
- SparkRowIterator iter, String[] names, byte[][] types);
-
- // for spark row -> ch columnar
- public static native void freeBlock(long blockAddress);
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockWriterJniWrapper.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockWriterJniWrapper.java
deleted file mode 100644
index 313515c13..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHBlockWriterJniWrapper.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-public class CHBlockWriterJniWrapper {
- private long instance = 0;
-
- private native long nativeCreateInstance();
-
- private native void nativeWrite(long instance, long block);
-
- private native int nativeResultSize(long instance);
-
- private native void nativeCollect(long instance, byte[] data);
-
- private native void nativeClose(long instance);
-
- public void write(ColumnarBatch columnarBatch) {
- if (instance == 0) {
- instance = nativeCreateInstance();
- }
- nativeWrite(instance, CHNativeBlock.fromColumnarBatch(columnarBatch).blockAddress());
- }
-
- public byte[] collectAsByteArray() {
- if (instance == 0L) {
- return new byte[0];
- }
- byte[] result = new byte[nativeResultSize(instance)];
- nativeCollect(instance, result);
- nativeClose(instance);
- instance = 0;
- return result;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHColumnVector.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHColumnVector.java
deleted file mode 100644
index 0b144b50f..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHColumnVector.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.vectorized.ColumnVector;
-import org.apache.spark.sql.vectorized.ColumnarArray;
-import org.apache.spark.sql.vectorized.ColumnarMap;
-import org.apache.spark.unsafe.types.UTF8String;
-
-public class CHColumnVector extends ColumnVector {
- private final int columnPosition;
- private long blockAddress;
-
- public CHColumnVector(DataType type, long blockAddress, int columnPosition) {
- super(type);
- this.blockAddress = blockAddress;
- this.columnPosition = columnPosition;
- }
-
- public long getBlockAddress() {
- return blockAddress;
- }
-
- @Override
- public void close() {
- // blockAddress = 0;
- }
-
- private native boolean nativeHasNull(long blockAddress, int columnPosition);
-
- @Override
- public boolean hasNull() {
- return nativeHasNull(blockAddress, columnPosition);
- }
-
- private native int nativeNumNulls(long blockAddress, int columnPosition);
-
- @Override
- public int numNulls() {
- return nativeNumNulls(blockAddress, columnPosition);
- }
-
- private native boolean nativeIsNullAt(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public boolean isNullAt(int rowId) {
- return nativeIsNullAt(rowId, blockAddress, columnPosition);
- }
-
- private native boolean nativeGetBoolean(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public boolean getBoolean(int rowId) {
- return nativeGetBoolean(rowId, blockAddress, columnPosition);
- }
-
- private native byte nativeGetByte(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public byte getByte(int rowId) {
- return nativeGetByte(rowId, blockAddress, columnPosition);
- }
-
- private native short nativeGetShort(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public short getShort(int rowId) {
- return nativeGetShort(rowId, blockAddress, columnPosition);
- }
-
- private native int nativeGetInt(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public int getInt(int rowId) {
- return nativeGetInt(rowId, blockAddress, columnPosition);
- }
-
- private native long nativeGetLong(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public long getLong(int rowId) {
- return nativeGetLong(rowId, blockAddress, columnPosition);
- }
-
- private native float nativeGetFloat(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public float getFloat(int rowId) {
- return nativeGetFloat(rowId, blockAddress, columnPosition);
- }
-
- private native double nativeGetDouble(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public double getDouble(int rowId) {
- return nativeGetDouble(rowId, blockAddress, columnPosition);
- }
-
- @Override
- public ColumnarArray getArray(int rowId) {
- return null;
- }
-
- @Override
- public ColumnarMap getMap(int ordinal) {
- return null;
- }
-
- @Override
- public Decimal getDecimal(int rowId, int precision, int scale) {
- return null;
- }
-
- private native String nativeGetString(int rowId, long blockAddress, int columnPosition);
-
- @Override
- public UTF8String getUTF8String(int rowId) {
- return UTF8String.fromString(nativeGetString(rowId, blockAddress, columnPosition));
- }
-
- @Override
- public byte[] getBinary(int rowId) {
- return new byte[0];
- }
-
- @Override
- public ColumnVector getChild(int ordinal) {
- return null;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeBlock.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeBlock.java
deleted file mode 100644
index 0f23e22bb..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeBlock.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.exception.GlutenException;
-
-import org.apache.spark.sql.execution.utils.CHExecUtil;
-import org.apache.spark.sql.vectorized.ColumnVector;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-public class CHNativeBlock {
- private long blockAddress;
-
- public CHNativeBlock(long blockAddress) {
- this.blockAddress = blockAddress;
- }
-
- public static CHNativeBlock fromColumnarBatch(ColumnarBatch batch) {
- if (batch.numCols() == 0 || !(batch.column(0) instanceof CHColumnVector)) {
- throw new GlutenException(
- "Unexpected ColumnarBatch: "
- + (batch.numCols() == 0
- ? "0 column"
- : "expected CHColumnVector, but " + batch.column(0).getClass()));
- }
- CHColumnVector columnVector = (CHColumnVector) batch.column(0);
- return new CHNativeBlock(columnVector.getBlockAddress());
- }
-
- private native int nativeNumRows(long blockAddress);
-
- public int numRows() {
- return nativeNumRows(blockAddress);
- }
-
- public long blockAddress() {
- return blockAddress;
- }
-
- private native int nativeNumColumns(long blockAddress);
-
- public int numColumns() {
- return nativeNumColumns(blockAddress);
- }
-
- private native byte[] nativeColumnType(long blockAddress, int position);
-
- public byte[] getTypeByPosition(int position) {
- return nativeColumnType(blockAddress, position);
- }
-
- private native long nativeTotalBytes(long blockAddress);
-
- public long totalBytes() {
- return nativeTotalBytes(blockAddress);
- }
-
- public native void nativeClose(long blockAddress);
-
- public void close() {
- if (blockAddress != 0) {
- nativeClose(blockAddress);
- blockAddress = 0;
- }
- }
-
- public static void closeFromColumnarBatch(ColumnarBatch cb) {
- if (cb != null) {
- if (cb.numCols() > 0) {
- CHColumnVector col = (CHColumnVector) cb.column(0);
- CHNativeBlock block = new CHNativeBlock(col.getBlockAddress());
- block.close();
- }
- cb.close();
- }
- }
-
- public ColumnarBatch toColumnarBatch() {
- ColumnVector[] vectors = new ColumnVector[numColumns()];
- for (int i = 0; i < numColumns(); i++) {
- vectors[i] =
- new CHColumnVector(CHExecUtil.inferSparkDataType(getTypeByPosition(i)), blockAddress, i);
- }
- int numRows = 0;
- if (numColumns() != 0) {
- numRows = numRows();
- }
- return new ColumnarBatch(vectors, numRows);
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeExpressionEvaluator.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeExpressionEvaluator.java
deleted file mode 100644
index fd33db769..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHNativeExpressionEvaluator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.GlutenConfig;
-import io.glutenproject.backendsapi.BackendsApiManager;
-import io.glutenproject.memory.alloc.CHNativeMemoryAllocators;
-import io.glutenproject.substrait.expression.ExpressionBuilder;
-import io.glutenproject.substrait.expression.StringMapNode;
-import io.glutenproject.substrait.extensions.AdvancedExtensionNode;
-import io.glutenproject.substrait.extensions.ExtensionBuilder;
-import io.glutenproject.substrait.plan.PlanBuilder;
-
-import org.apache.spark.SparkConf;
-import org.apache.spark.sql.internal.SQLConf;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import scala.Tuple2;
-import scala.collection.JavaConverters;
-
-public class CHNativeExpressionEvaluator {
- private final ExpressionEvaluatorJniWrapper jniWrapper;
-
- public CHNativeExpressionEvaluator() {
- jniWrapper = new ExpressionEvaluatorJniWrapper();
- }
-
- // Used to initialize the native computing.
- public void initNative(SparkConf conf) {
- Tuple2<String, String>[] all = conf.getAll();
- Map<String, String> confMap =
- Arrays.stream(all).collect(Collectors.toMap(Tuple2::_1, Tuple2::_2));
- String prefix = BackendsApiManager.getSettings().getBackendConfigPrefix();
- Map<String, String> nativeConfMap =
- GlutenConfig.getNativeBackendConf(prefix, JavaConverters.mapAsScalaMap(confMap));
-
- // Get the customer config from SparkConf for each backend
- BackendsApiManager.getTransformerApiInstance().postProcessNativeConfig(nativeConfMap, prefix);
-
- jniWrapper.nativeInitNative(buildNativeConf(nativeConfMap));
- }
-
- public void finalizeNative() {
- jniWrapper.nativeFinalizeNative();
- }
-
- // Used to validate the Substrait plan in native compute engine.
- public boolean doValidate(byte[] subPlan) {
- return jniWrapper.nativeDoValidate(subPlan);
- }
-
- private byte[] buildNativeConf(Map<String, String> confs) {
- StringMapNode stringMapNode = ExpressionBuilder.makeStringMap(confs);
- AdvancedExtensionNode extensionNode =
- ExtensionBuilder.makeAdvancedExtension(
- BackendsApiManager.getTransformerApiInstance()
- .packPBMessage(stringMapNode.toProtobuf()));
- return PlanBuilder.makePlan(extensionNode).toProtobuf().toByteArray();
- }
-
- private Map<String, String> getNativeBackendConf() {
- return GlutenConfig.getNativeBackendConf(
- BackendsApiManager.getSettings().getBackendConfigPrefix(), SQLConf.get().getAllConfs());
- }
-
- // Used by WholeStageTransform to create the native computing pipeline and
- // return a columnar result iterator.
- public GeneralOutIterator createKernelWithBatchIterator(
- byte[] wsPlan,
- byte[][] splitInfo,
- List<GeneralInIterator> iterList,
- boolean materializeInput) {
- long allocId = CHNativeMemoryAllocators.contextInstance().getNativeInstanceId();
- long handle =
- jniWrapper.nativeCreateKernelWithIterator(
- allocId,
- wsPlan,
- splitInfo,
- iterList.toArray(new GeneralInIterator[0]),
- buildNativeConf(getNativeBackendConf()),
- materializeInput);
- return createOutIterator(handle);
- }
-
- // Only for UT.
- public GeneralOutIterator createKernelWithBatchIterator(
- long allocId, byte[] wsPlan, byte[][] splitInfo, List<GeneralInIterator> iterList) {
- long handle =
- jniWrapper.nativeCreateKernelWithIterator(
- allocId,
- wsPlan,
- splitInfo,
- iterList.toArray(new GeneralInIterator[0]),
- buildNativeConf(getNativeBackendConf()),
- false);
- return createOutIterator(handle);
- }
-
- private GeneralOutIterator createOutIterator(long nativeHandle) {
- return new BatchIterator(nativeHandle);
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHShuffleSplitterJniWrapper.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHShuffleSplitterJniWrapper.java
deleted file mode 100644
index af4ab1ecf..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHShuffleSplitterJniWrapper.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import java.io.IOException;
-
-public class CHShuffleSplitterJniWrapper {
- public CHShuffleSplitterJniWrapper() {}
-
- public long make(
- NativePartitioning part,
- int shuffleId,
- long mapId,
- int bufferSize,
- String codec,
- String dataFile,
- String localDirs,
- int subDirsPerLocalDir,
- boolean preferSpill,
- long spillThreshold,
- String hashAlgorithm,
- boolean throwIfMemoryExceed,
- boolean flushBlockBufferBeforeEvict) {
- return nativeMake(
- part.getShortName(),
- part.getNumPartitions(),
- part.getExprList(),
- part.getRequiredFields(),
- shuffleId,
- mapId,
- bufferSize,
- codec,
- dataFile,
- localDirs,
- subDirsPerLocalDir,
- preferSpill,
- spillThreshold,
- hashAlgorithm,
- throwIfMemoryExceed,
- flushBlockBufferBeforeEvict);
- }
-
- public long makeForRSS(
- NativePartitioning part,
- int shuffleId,
- long mapId,
- int bufferSize,
- String codec,
- long spillThreshold,
- String hashAlgorithm,
- Object pusher,
- boolean throwIfMemoryExceed,
- boolean flushBlockBufferBeforeEvict) {
- return nativeMakeForRSS(
- part.getShortName(),
- part.getNumPartitions(),
- part.getExprList(),
- part.getRequiredFields(),
- shuffleId,
- mapId,
- bufferSize,
- codec,
- spillThreshold,
- hashAlgorithm,
- pusher,
- throwIfMemoryExceed,
- flushBlockBufferBeforeEvict);
- }
-
- public native long nativeMake(
- String shortName,
- int numPartitions,
- byte[] exprList,
- byte[] exprIndexList,
- int shuffleId,
- long mapId,
- int bufferSize,
- String codec,
- String dataFile,
- String localDirs,
- int subDirsPerLocalDir,
- boolean preferSpill,
- long spillThreshold,
- String hashAlgorithm,
- boolean throwIfMemoryExceed,
- boolean flushBlockBufferBeforeEvict);
-
- public native long nativeMakeForRSS(
- String shortName,
- int numPartitions,
- byte[] exprList,
- byte[] exprIndexList,
- int shuffleId,
- long mapId,
- int bufferSize,
- String codec,
- long spillThreshold,
- String hashAlgorithm,
- Object pusher,
- boolean throwIfMemoryExceed,
- boolean flushBlockBufferBeforeEvict);
-
- public native void split(long splitterId, long block);
-
- public native long evict(long splitterId);
-
- public native CHSplitResult stop(long splitterId) throws IOException;
-
- public native void close(long splitterId);
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHStreamReader.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHStreamReader.java
deleted file mode 100644
index e7497c9de..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/CHStreamReader.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.backendsapi.clickhouse.CHBackendSettings;
-
-import org.apache.spark.storage.CHShuffleReadStreamFactory;
-
-import java.io.InputStream;
-
-public class CHStreamReader implements AutoCloseable {
- private final ShuffleInputStream inputStream;
- private long nativeShuffleReader;
-
- public CHStreamReader(
- InputStream inputStream, boolean forceCompress, boolean isCustomizedShuffleCodec) {
- this(CHShuffleReadStreamFactory.create(inputStream, forceCompress, isCustomizedShuffleCodec));
- }
-
- public CHStreamReader(ShuffleInputStream shuffleInputStream) {
- inputStream = shuffleInputStream;
- nativeShuffleReader =
- createNativeShuffleReader(
- this.inputStream,
- inputStream.isCompressed(),
- CHBackendSettings.maxShuffleReadRows(),
- CHBackendSettings.maxShuffleReadBytes());
- }
-
- private static native long createNativeShuffleReader(
- ShuffleInputStream inputStream,
- boolean compressed,
- long maxShuffleReadRows,
- long maxShuffleReadBytes);
-
- private native long nativeNext(long nativeShuffleReader);
-
- public CHNativeBlock next() {
- long block = nativeNext(nativeShuffleReader);
- return new CHNativeBlock(block);
- }
-
- private native void nativeClose(long shuffleReader);
-
- @Override
- public void close() throws Exception {
- // close input stream and release buffer
- this.inputStream.close();
- nativeClose(nativeShuffleReader);
- nativeShuffleReader = 0L;
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ExpressionEvaluatorJniWrapper.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ExpressionEvaluatorJniWrapper.java
deleted file mode 100644
index 2fd281b28..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ExpressionEvaluatorJniWrapper.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-/**
- * This class is implemented in JNI. This provides the Java interface to invoke functions in JNI.
- * This file is used to generate the .h files required for jni. Avoid all external dependencies in
- * this file.
- */
-public class ExpressionEvaluatorJniWrapper {
-
- /** Call initNative to initialize native computing. */
- native void nativeInitNative(byte[] confAsPlan);
-
- /** Call finalizeNative to finalize native computing. */
- native void nativeFinalizeNative();
-
- /**
- * Validate the Substrait plan in native compute engine.
- *
- * @param subPlan the Substrait plan in binary format.
- * @return whether the computing of this plan is supported in native.
- */
- native boolean nativeDoValidate(byte[] subPlan);
-
- /**
- * Create a native compute kernel and return a columnar result iterator.
- *
- * @param allocatorId allocator id
- * @return iterator instance id
- */
- public native long nativeCreateKernelWithIterator(
- long allocatorId,
- byte[] wsPlan,
- byte[][] splitInfo,
- GeneralInIterator[] batchItr,
- byte[] confArray,
- boolean materializeInput);
-
- /**
- * Closes the projector referenced by nativeHandler.
- *
- * @param nativeHandler nativeHandler that needs to be closed
- */
- native void nativeClose(long nativeHandler);
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/IteratorWrapper.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/IteratorWrapper.java
deleted file mode 100644
index 8abdac18e..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/IteratorWrapper.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import java.util.Iterator;
-
-public class IteratorWrapper {
- private final Iterator<Long> in;
-
- public IteratorWrapper(Iterator<Long> in) {
- this.in = in;
- }
-
- public boolean hasNext() {
- return in.hasNext();
- }
-
- public long next() {
- return in.next();
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyFileSegmentShuffleInputStream.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyFileSegmentShuffleInputStream.java
deleted file mode 100644
index 3b271f2b5..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyFileSegmentShuffleInputStream.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.exception.GlutenException;
-
-import io.netty.util.internal.PlatformDependent;
-import org.apache.spark.network.util.LimitedInputStream;
-import org.apache.spark.storage.CHShuffleReadStreamFactory;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-
-public class LowCopyFileSegmentShuffleInputStream implements ShuffleInputStream {
-
- private final InputStream in;
- private final FileChannel channel;
- private final boolean isCompressed;
-
- private long bytesRead = 0L;
- private long left;
-
- public LowCopyFileSegmentShuffleInputStream(
- InputStream in, LimitedInputStream limitedInputStream, boolean isCompressed) {
- // to prevent underlying netty buffer from being collected by GC
- this.in = in;
- this.isCompressed = isCompressed;
- final FileInputStream fin;
- try {
- left =
- ((long) CHShuffleReadStreamFactory.FIELD_LimitedInputStream_left.get(limitedInputStream));
- fin =
- (FileInputStream)
- CHShuffleReadStreamFactory.FIELD_FilterInputStream_in.get(limitedInputStream);
- } catch (IllegalAccessException e) {
- throw new GlutenException(e);
- }
- channel = fin.getChannel();
- }
-
- @Override
- public long read(long destAddress, long maxReadSize) {
- long bytesToRead = Math.min(left, maxReadSize);
- int bytesToRead32 = Math.toIntExact(bytesToRead);
- if (bytesToRead32 == 0) {
- return 0;
- }
- ByteBuffer direct = PlatformDependent.directBuffer(destAddress, bytesToRead32);
- try {
- // read data from file channel to native address
- int bytes = channel.read(direct);
- if (bytes == -1) {
- return 0;
- }
- bytesRead += bytes;
- left -= bytes;
- return bytes;
- } catch (IOException e) {
- throw new GlutenException(e);
- }
- }
-
- @Override
- public long pos() {
- return bytesRead;
- }
-
- @Override
- public boolean isCompressed() {
- return this.isCompressed;
- }
-
- @Override
- public void close() {
- GlutenException.wrap(
- () -> {
- channel.close();
- in.close();
- return null;
- });
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyNettyShuffleInputStream.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyNettyShuffleInputStream.java
deleted file mode 100644
index 27b2c5b85..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/LowCopyNettyShuffleInputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.exception.GlutenException;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-public class LowCopyNettyShuffleInputStream implements ShuffleInputStream {
-
- private final InputStream in;
- private final boolean isCompressed;
-
- private final ByteBuf byteBuf;
- private int readBytesCount = 0;
-
- public LowCopyNettyShuffleInputStream(InputStream in, ByteBuf byteBuf, boolean isCompressed) {
- // to prevent underlying netty buffer from being collected by GC
- this.in = in;
- this.byteBuf = byteBuf;
- this.isCompressed = isCompressed;
- }
-
- @Override
- public long read(long destAddress, long maxReadSize) {
- long bytesToRead = Math.min(maxReadSize, this.byteBuf.readableBytes());
- int bytesToRead32 = Math.toIntExact(bytesToRead);
- if (bytesToRead32 == 0) {
- return 0;
- }
- ByteBuffer direct = PlatformDependent.directBuffer(destAddress, bytesToRead32);
- // read data directly from ByteBuf to native address
- this.byteBuf.readBytes(direct);
- readBytesCount += direct.position();
- return direct.position();
- }
-
- @Override
- public long pos() {
- return readBytesCount;
- }
-
- @Override
- public boolean isCompressed() {
- return this.isCompressed;
- }
-
- @Override
- public void close() {
- GlutenException.wrap(
- () -> {
- in.close();
- return null;
- });
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/OnHeapCopyShuffleInputStream.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/OnHeapCopyShuffleInputStream.java
deleted file mode 100644
index 03bf60534..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/OnHeapCopyShuffleInputStream.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.exception.GlutenException;
-
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.InputStream;
-
-public class OnHeapCopyShuffleInputStream implements ShuffleInputStream {
-
- private InputStream in;
- private final boolean isCompressed;
- private long bytesRead = 0L;
-
- private byte[] buffer;
-
- public OnHeapCopyShuffleInputStream(InputStream in, boolean isCompressed) {
- this.in = in;
- this.isCompressed = isCompressed;
- }
-
- @Override
- public long read(long destAddress, long maxReadSize) {
- return GlutenException.wrap(
- () -> {
- int maxReadSize32 = Math.toIntExact(maxReadSize);
- if (buffer == null || maxReadSize32 > buffer.length) {
- this.buffer = new byte[maxReadSize32];
- }
- // The code conducts copy as long as 'in' wraps off-heap data,
- // which is about to be moved to heap
- int read = in.read(buffer, 0, maxReadSize32);
- if (read == -1 || read == 0) {
- return 0;
- }
- // The code conducts copy, from heap to off-heap
- // memCopyFromHeap(buffer, destAddress, read);
- PlatformDependent.copyMemory(buffer, 0, destAddress, read);
- bytesRead += read;
- return read;
- });
- }
-
- @Override
- public long pos() {
- return bytesRead;
- }
-
- @Override
- public boolean isCompressed() {
- return this.isCompressed;
- }
-
- @Override
- public void close() {
- GlutenException.wrap(
- () -> {
- in.close();
- in = null;
- return null;
- });
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ShuffleInputStream.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ShuffleInputStream.java
deleted file mode 100644
index 9ce7896c1..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/ShuffleInputStream.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-public interface ShuffleInputStream {
-
- /**
- * Read fixed size of data into an offheap address.
- *
- * @return the read bytes; 0 if end of stream.
- */
- long read(long destAddress, long maxReadSize);
-
- boolean isCompressed();
-
- /** Position of this stream. */
- long pos();
-
- /** Close and reclaim the resources. */
- void close();
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/SimpleExpressionEval.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/SimpleExpressionEval.java
deleted file mode 100644
index 5719b194f..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/SimpleExpressionEval.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.execution.ColumnarNativeIterator;
-import io.glutenproject.substrait.plan.PlanNode;
-
-import io.substrait.proto.Plan;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-
-public class SimpleExpressionEval implements AutoCloseable, Iterator<Long> {
-
- private static final Logger LOG = LoggerFactory.getLogger(SimpleExpressionEval.class);
-
- private final Long instance;
-
- public SimpleExpressionEval(ColumnarNativeIterator blockStream, PlanNode planNode) {
- Plan plan = planNode.toProtobuf();
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("SimpleExpressionEval exec plan: %s", plan.toString()));
- }
- byte[] planData = plan.toByteArray();
- instance = createNativeInstance(blockStream, planData);
- }
-
- private static native long createNativeInstance(ColumnarNativeIterator blockStream, byte[] plan);
-
- private static native void nativeClose(long instance);
-
- private static native boolean nativeHasNext(long instance);
-
- private static native long nativeNext(long instance);
-
- @Override
- public boolean hasNext() {
- return nativeHasNext(instance);
- }
-
- @Override
- public Long next() {
- return nativeNext(instance);
- }
-
- @Override
- public void close() throws Exception {
- nativeClose(instance);
- }
-}
diff --git a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/StorageJoinBuilder.java b/backends-clickhouse/src/main/java/io/glutenproject/vectorized/StorageJoinBuilder.java
deleted file mode 100644
index 91d05a93d..000000000
--- a/backends-clickhouse/src/main/java/io/glutenproject/vectorized/StorageJoinBuilder.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.vectorized;
-
-import io.glutenproject.execution.BroadCastHashJoinContext;
-import io.glutenproject.expression.ConverterUtils;
-import io.glutenproject.expression.ConverterUtils$;
-import io.glutenproject.substrait.type.TypeNode;
-import io.glutenproject.utils.SubstraitUtil;
-
-import io.substrait.proto.NamedStruct;
-import io.substrait.proto.Type;
-import org.apache.spark.sql.catalyst.expressions.Attribute;
-import org.apache.spark.sql.catalyst.expressions.Expression;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import scala.collection.JavaConverters;
-
-public class StorageJoinBuilder {
-
- public static native void nativeCleanBuildHashTable(String hashTableId, long hashTableData);
-
- public static native long nativeCloneBuildHashTable(long hashTableData);
-
- private static native long nativeBuild(
- String buildHashTableId,
- byte[] in,
- long rowCount,
- String joinKeys,
- int joinType,
- byte[] namedStruct);
-
- private StorageJoinBuilder() {}
-
- /** build storage join object */
- public static long build(
- byte[] batches,
- long rowCount,
- BroadCastHashJoinContext broadCastContext,
- List<Expression> newBuildKeys,
- List<Attribute> newOutput) {
- ConverterUtils$ converter = ConverterUtils$.MODULE$;
- List<Expression> keys;
- List<Attribute> output;
- if (newBuildKeys.isEmpty()) {
- keys = JavaConverters.<Expression>seqAsJavaList(broadCastContext.buildSideJoinKeys());
- output = JavaConverters.<Attribute>seqAsJavaList(broadCastContext.buildSideStructure());
- } else {
- keys = newBuildKeys;
- output = newOutput;
- }
- String joinKey =
- keys.stream()
- .map(
- (Expression key) -> {
- Attribute attr = converter.getAttrFromExpr(key);
- return converter.genColumnNameWithExprId(attr);
- })
- .collect(Collectors.joining(","));
- return nativeBuild(
- broadCastContext.buildHashTableId(),
- batches,
- rowCount,
- joinKey,
- SubstraitUtil.toSubstrait(broadCastContext.joinType()).ordinal(),
- toNameStruct(output).toByteArray());
- }
-
- /** create table named struct */
- private static NamedStruct toNameStruct(List<Attribute> output) {
- List<TypeNode> typeList = ConverterUtils.collectAttributeTypeNodes(output);
- List<String> nameList = ConverterUtils.collectAttributeNamesWithExprId(output);
- Type.Struct.Builder structBuilder = Type.Struct.newBuilder();
- for (TypeNode typeNode : typeList) {
- structBuilder.addTypes(typeNode.toProtobuf());
- }
- NamedStruct.Builder nStructBuilder = NamedStruct.newBuilder();
- nStructBuilder.setStruct(structBuilder.build());
- for (String name : nameList) {
- nStructBuilder.addNames(name);
- }
- return nStructBuilder.build();
- }
-}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/execution/ColumnarNativeIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/execution/ColumnarNativeIterator.java
new file mode 100644
index 000000000..2a7446711
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/execution/ColumnarNativeIterator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution;
+
+import org.apache.gluten.vectorized.CHColumnVector;
+import org.apache.gluten.vectorized.GeneralInIterator;
+
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+import java.util.Iterator;
+
+public class ColumnarNativeIterator extends GeneralInIterator implements Iterator<byte[]> {
+
+ public ColumnarNativeIterator(Iterator<ColumnarBatch> delegated) {
+ super(delegated);
+ }
+
+ private transient ColumnarBatch nextBatch = null;
+
+ private static byte[] longtoBytes(long data) {
+ return new byte[] {
+ (byte) ((data >> 56) & 0xff),
+ (byte) ((data >> 48) & 0xff),
+ (byte) ((data >> 40) & 0xff),
+ (byte) ((data >> 32) & 0xff),
+ (byte) ((data >> 24) & 0xff),
+ (byte) ((data >> 16) & 0xff),
+ (byte) ((data >> 8) & 0xff),
+ (byte) ((data >> 0) & 0xff),
+ };
+ }
+
+ @Override
+ public boolean hasNext() {
+ while (delegated.hasNext()) {
+ nextBatch = delegated.next();
+ if (nextBatch.numRows() > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public ColumnarBatch nextColumnarBatch() {
+ return nextBatch;
+ }
+
+ @Override
+ public byte[] next() {
+ ColumnarBatch nextBatch = nextColumnarBatch();
+ if (nextBatch.numRows() > 0) {
+ CHColumnVector col = (CHColumnVector) nextBatch.column(0);
+ return longtoBytes(col.getBlockAddress());
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/execution/SparkRowIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/execution/SparkRowIterator.java
new file mode 100644
index 000000000..75a133460
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/execution/SparkRowIterator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Iterator;
+
+public class SparkRowIterator implements Iterator<byte[]> {
+ private final scala.collection.Iterator<byte[]> delegated;
+ private final int defaultBufSize = 4096;
+ private byte[] lastRowBuf;
+
+ public SparkRowIterator(scala.collection.Iterator<byte[]> delegated) {
+ this.delegated = delegated;
+ lastRowBuf = null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return lastRowBuf != null || delegated.hasNext();
+ }
+
+ @Override
+ public byte[] next() {
+ return delegated.next();
+ }
+
+ protected ByteBuffer createByteBuffer(int bufSize) {
+ ByteBuffer buf;
+ // 8: one length and one end flag
+ if (bufSize + 8 > defaultBufSize) {
+ buf = ByteBuffer.allocateDirect(bufSize + 8);
+ } else {
+ buf = ByteBuffer.allocateDirect(defaultBufSize);
+ }
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ return buf;
+ }
+
+ public ByteBuffer nextBatch() {
+ ByteBuffer buf = null;
+ if (lastRowBuf != null) {
+ buf = createByteBuffer(lastRowBuf.length);
+ buf.putInt(lastRowBuf.length);
+ buf.put(lastRowBuf);
+ // make the end flag
+ buf.putInt(-1);
+ lastRowBuf = null;
+ return buf;
+ } else {
+ while (delegated.hasNext()) {
+ lastRowBuf = delegated.next();
+ if (buf == null) {
+ buf = createByteBuffer(lastRowBuf.length);
+ }
+ if (buf.remaining() < lastRowBuf.length + 8) {
+ break;
+ } else {
+ buf.putInt(lastRowBuf.length);
+ buf.put(lastRowBuf);
+ lastRowBuf = null;
+ }
+ }
+ buf.putInt(-1);
+ return buf;
+ }
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHManagedCHReservationListener.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHManagedCHReservationListener.java
new file mode 100644
index 000000000..1f560a905
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHManagedCHReservationListener.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+import org.apache.gluten.GlutenConfig;
+import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
+import org.apache.gluten.memory.memtarget.MemoryTarget;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class CHManagedCHReservationListener implements CHReservationListener {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CHManagedCHReservationListener.class);
+
+ private MemoryTarget target;
+ private final SimpleMemoryUsageRecorder usage;
+ private final boolean throwIfMemoryExceed =
+ GlutenConfig.getConf().chColumnarThrowIfMemoryExceed();
+ private volatile boolean open = true;
+
+ private final AtomicLong currentMemory = new AtomicLong(0L);
+
+ public CHManagedCHReservationListener(MemoryTarget target, SimpleMemoryUsageRecorder usage) {
+ this.target = target;
+ this.usage = usage;
+ }
+
+ @Override
+ public void reserveOrThrow(long size) {
+ if (!throwIfMemoryExceed) {
+ reserve(size);
+ return;
+ }
+
+ synchronized (this) {
+ if (!open) {
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("reserve memory size from native: %d", size));
+ }
+ long granted = target.borrow(size);
+ if (granted < size) {
+ target.repay(granted);
+ throw new UnsupportedOperationException(
+ "Not enough spark off-heap execution memory. "
+ + "Acquired: "
+ + size
+ + ", granted: "
+ + granted
+ + ". "
+ + "Try tweaking config option spark.memory.offHeap.size to "
+ + "get larger space to run this application. ");
+ }
+ currentMemory.addAndGet(size);
+ usage.inc(size);
+ }
+ }
+
+ @Override
+ public long reserve(long size) {
+ synchronized (this) {
+ if (!open) {
+ return 0L;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("reserve memory (without exception) size from native: %d", size));
+ }
+ long granted = target.borrow(size);
+ if (granted < size && (LOG.isWarnEnabled())) {
+ LOG.warn(
+ String.format(
+ "Not enough spark off-heap execution memory. "
+ + "Acquired: %d, granted: %d. Try tweaking config option "
+ + "spark.memory.offHeap.size to get larger space "
+ + "to run this application.",
+ size, granted));
+ }
+ currentMemory.addAndGet(granted);
+ usage.inc(size);
+ return granted;
+ }
+ }
+
+ @Override
+ public long unreserve(long size) {
+ synchronized (this) {
+ if (!open) {
+ return 0L;
+ }
+ long memoryToFree = size;
+ if ((currentMemory.get() - size) < 0L) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ String.format(
+ "The current used memory' %d will be less than 0(%d) after free %d",
+ currentMemory.get(), currentMemory.get() - size, size));
+ }
+ memoryToFree = currentMemory.get();
+ }
+ if (memoryToFree == 0L) {
+ return memoryToFree;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("unreserve memory size from native: %d", memoryToFree));
+ }
+ target.repay(memoryToFree);
+ currentMemory.addAndGet(-memoryToFree);
+ usage.inc(-size);
+ return memoryToFree;
+ }
+ }
+
+ @Override
+ public void inactivate() {
+ synchronized (this) {
+ // for some reasons, memory audit in the native code may not be 100% accurate
+ // we'll allow the inaccuracy
+ if (currentMemory.get() > 0) {
+ unreserve(currentMemory.get());
+ } else if (currentMemory.get() < 0) {
+ reserve(currentMemory.get());
+ }
+ currentMemory.set(0L);
+
+ target = null; // make it gc reachable
+ open = false;
+ }
+ }
+
+ @Override
+ public long currentMemory() {
+ return currentMemory.get();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocator.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocator.java
new file mode 100644
index 000000000..0df0757c8
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+/**
+ * Like {@link org.apache.gluten.vectorized.NativePlanEvaluator}, this along with {@link
+ * CHNativeMemoryAllocators}, as built-in toolkit for managing native memory allocations.
+ */
+public class CHNativeMemoryAllocator {
+
+ private final long nativeInstanceId;
+ private final CHReservationListener listener;
+
+ public CHNativeMemoryAllocator(long nativeInstanceId, CHReservationListener listener) {
+ this.nativeInstanceId = nativeInstanceId;
+ this.listener = listener;
+ }
+
+ public static CHNativeMemoryAllocator getDefault() {
+ return new CHNativeMemoryAllocator(getDefaultAllocator(), CHReservationListener.NOOP);
+ }
+
+ public static CHNativeMemoryAllocator getDefaultForUT() {
+ return new CHNativeMemoryAllocator(
+ createListenableAllocator(CHReservationListener.NOOP), CHReservationListener.NOOP);
+ }
+
+ public static CHNativeMemoryAllocator createListenable(CHReservationListener listener) {
+ return new CHNativeMemoryAllocator(createListenableAllocator(listener), listener);
+ }
+
+ public CHReservationListener listener() {
+ return listener;
+ }
+
+ public long getNativeInstanceId() {
+ return this.nativeInstanceId;
+ }
+
+ public long getBytesAllocated() {
+ if (this.nativeInstanceId == -1L) return 0;
+ return bytesAllocated(this.nativeInstanceId);
+ }
+
+ public void close() {
+ if (this.nativeInstanceId == -1L) return;
+ releaseAllocator(this.nativeInstanceId);
+ }
+
+ private static native long getDefaultAllocator();
+
+ private static native long createListenableAllocator(CHReservationListener listener);
+
+ private static native void releaseAllocator(long allocatorId);
+
+ private static native long bytesAllocated(long allocatorId);
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManager.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManager.java
new file mode 100644
index 000000000..6645748e5
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManager.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+import org.apache.spark.util.TaskResource;
+
+/** Resource manager implementation that manages a {@link CHNativeMemoryAllocator}. */
+public interface CHNativeMemoryAllocatorManager extends TaskResource {
+ CHNativeMemoryAllocator getManaged();
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java
new file mode 100644
index 000000000..2c1c2547f
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocatorManagerImpl.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+public class CHNativeMemoryAllocatorManagerImpl implements CHNativeMemoryAllocatorManager {
+ private final CHNativeMemoryAllocator managed;
+
+ public CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator managed) {
+ this.managed = managed;
+ }
+
+ @Override
+ public void release() {
+ managed.close();
+ managed.listener().inactivate();
+ }
+
+ @Override
+ public CHNativeMemoryAllocator getManaged() {
+ return managed;
+ }
+
+ @Override
+ public int priority() {
+ return 0; // lowest priority
+ }
+
+ @Override
+ public String resourceName() {
+ return "CHNativeMemoryAllocatorManager_" + managed.getNativeInstanceId();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java
new file mode 100644
index 000000000..01fb7e3e2
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
+import org.apache.gluten.memory.memtarget.MemoryTargets;
+import org.apache.gluten.memory.memtarget.Spiller;
+
+import org.apache.spark.memory.TaskMemoryManager;
+import org.apache.spark.util.TaskResources;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Built-in toolkit for managing native memory allocations. To use the facility, one should import
+ * Gluten's C++ library then create the c++ instance using following example code:
+ *
+ * <p>```c++ auto* allocator = reinterpret_cast<gluten::memory::MemoryAllocator*>(allocator_id); ```
+ *
+ * <p>The ID "allocator_id" can be retrieved from Java API {@link
+ * CHNativeMemoryAllocator#getNativeInstanceId()}.
+ *
+ * <p>FIXME: to export the native APIs in a standard way
+ */
+public abstract class CHNativeMemoryAllocators {
+ private CHNativeMemoryAllocators() {}
+
+ private static final CHNativeMemoryAllocator GLOBAL = CHNativeMemoryAllocator.getDefault();
+
+ private static CHNativeMemoryAllocatorManager createNativeMemoryAllocatorManager(
+ String name,
+ TaskMemoryManager taskMemoryManager,
+ List<Spiller> spillers,
+ SimpleMemoryUsageRecorder usage) {
+
+ CHManagedCHReservationListener rl =
+ new CHManagedCHReservationListener(
+ MemoryTargets.newConsumer(taskMemoryManager, name, spillers, Collections.emptyMap()),
+ usage);
+ return new CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator.createListenable(rl));
+ }
+
+ public static CHNativeMemoryAllocator contextInstance() {
+ if (!TaskResources.inSparkTask()) {
+ return globalInstance();
+ }
+
+ final String id = CHNativeMemoryAllocatorManager.class.toString();
+ if (!TaskResources.isResourceRegistered(id)) {
+ final CHNativeMemoryAllocatorManager manager =
+ createNativeMemoryAllocatorManager(
+ "ContextInstance",
+ TaskResources.getLocalTaskContext().taskMemoryManager(),
+ Collections.emptyList(),
+ TaskResources.getSharedUsage());
+ TaskResources.addResource(id, manager);
+ }
+ return ((CHNativeMemoryAllocatorManager) TaskResources.getResource(id)).getManaged();
+ }
+
+ public static CHNativeMemoryAllocator contextInstanceForUT() {
+ return CHNativeMemoryAllocator.getDefaultForUT();
+ }
+
+ public static CHNativeMemoryAllocator createSpillable(String name, Spiller... spillers) {
+ if (!TaskResources.inSparkTask()) {
+ throw new IllegalStateException("spiller must be used in a Spark task");
+ }
+
+ final CHNativeMemoryAllocatorManager manager =
+ createNativeMemoryAllocatorManager(
+ name,
+ TaskResources.getLocalTaskContext().taskMemoryManager(),
+ Arrays.asList(spillers),
+ TaskResources.getSharedUsage());
+ TaskResources.addAnonymousResource(manager);
+ // force add memory consumer to task memory manager, will release by inactivate
+ manager.getManaged().listener().reserve(1);
+ return manager.getManaged();
+ }
+
+ public static CHNativeMemoryAllocator globalInstance() {
+ return GLOBAL;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHReservationListener.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHReservationListener.java
new file mode 100644
index 000000000..926c4426d
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHReservationListener.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.alloc;
+
+public interface CHReservationListener {
+ CHReservationListener NOOP =
+ new CHReservationListener() {
+ @Override
+ public void reserveOrThrow(long size) {}
+
+ @Override
+ public long reserve(long size) {
+ return 0L;
+ }
+
+ @Override
+ public long unreserve(long size) {
+ return 0L;
+ }
+
+ @Override
+ public void inactivate() {}
+
+ @Override
+ public long currentMemory() {
+ return 0L;
+ }
+ };
+
+ long reserve(long size);
+
+ void reserveOrThrow(long size);
+
+ long unreserve(long size);
+
+ void inactivate();
+
+ long currentMemory();
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsData.java b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsData.java
new file mode 100644
index 000000000..fd31b2210
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsData.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+public class MetricsData {
+
+ protected long id;
+ protected String name;
+ protected long time;
+
+ @JsonProperty("input_wait_time")
+ protected long inputWaitTime;
+
+ @JsonProperty("output_wait_time")
+ protected long outputWaitTime;
+
+ protected long inputRows = 0;
+ protected long inputVectors = 0;
+ protected long inputBytes = 0;
+ protected long outputRows = 0;
+ protected long outputVectors = 0;
+ protected long outputBytes = 0;
+ protected List<MetricsStep> steps;
+
+ public long getId() {
+ return id;
+ }
+
+ public void setId(long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public long getTime() {
+ return time;
+ }
+
+ public void setTime(long time) {
+ this.time = time;
+ }
+
+ public List<MetricsStep> getSteps() {
+ return steps;
+ }
+
+ public void setSteps(List<MetricsStep> steps) {
+ this.steps = steps;
+ }
+
+ public long getInputRows() {
+ return inputRows;
+ }
+
+ public void setInputRows(long inputRows) {
+ this.inputRows = inputRows;
+ }
+
+ public long getInputVectors() {
+ return inputVectors;
+ }
+
+ public void setInputVectors(long inputVectors) {
+ this.inputVectors = inputVectors;
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ public void setInputBytes(long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public long getOutputRows() {
+ return outputRows;
+ }
+
+ public void setOutputRows(long outputRows) {
+ this.outputRows = outputRows;
+ }
+
+ public long getOutputVectors() {
+ return outputVectors;
+ }
+
+ public void setOutputVectors(long outputVectors) {
+ this.outputVectors = outputVectors;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ public void setOutputBytes(long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+
+ public long getInputWaitTime() {
+ return inputWaitTime;
+ }
+
+ public void setInputWaitTime(long inputWaitTime) {
+ this.inputWaitTime = inputWaitTime;
+ }
+
+ public long getOutputWaitTime() {
+ return outputWaitTime;
+ }
+
+ public void setOutputWaitTime(long outputWaitTime) {
+ this.outputWaitTime = outputWaitTime;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsProcessor.java b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsProcessor.java
new file mode 100644
index 000000000..18da9eef8
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsProcessor.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class MetricsProcessor {
+
+ protected String name;
+ protected long time;
+
+ @JsonProperty("input_rows")
+ protected long inputRows = 0;
+
+ protected long inputVectors = 0;
+
+ @JsonProperty("input_bytes")
+ protected long inputBytes = 0;
+
+ @JsonProperty("output_rows")
+ protected long outputRows = 0;
+
+ protected long outputVectors = 0;
+
+ @JsonProperty("output_bytes")
+ protected long outputBytes = 0;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public long getTime() {
+ return time;
+ }
+
+ public void setTime(long time) {
+ this.time = time;
+ }
+
+ public long getInputRows() {
+ return inputRows;
+ }
+
+ public void setInputRows(long inputRows) {
+ this.inputRows = inputRows;
+ }
+
+ public long getInputVectors() {
+ return inputVectors;
+ }
+
+ public void setInputVectors(long inputVectors) {
+ this.inputVectors = inputVectors;
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ public void setInputBytes(long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public long getOutputRows() {
+ return outputRows;
+ }
+
+ public void setOutputRows(long outputRows) {
+ this.outputRows = outputRows;
+ }
+
+ public long getOutputVectors() {
+ return outputVectors;
+ }
+
+ public void setOutputVectors(long outputVectors) {
+ this.outputVectors = outputVectors;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ public void setOutputBytes(long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsStep.java b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsStep.java
new file mode 100644
index 000000000..f95aaa323
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/MetricsStep.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+public class MetricsStep {
+
+ protected String name;
+ protected String description;
+ protected List<MetricsProcessor> processors;
+
+ @JsonProperty("total_marks_pk")
+ protected long totalMarksPk;
+
+ @JsonProperty("selected_marks_pk")
+ protected long selectedMarksPk;
+
+ @JsonProperty("selected_marks")
+ protected long selectedMarks;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public List<MetricsProcessor> getProcessors() {
+ return processors;
+ }
+
+ public void setProcessors(List<MetricsProcessor> processors) {
+ this.processors = processors;
+ }
+
+ public void setTotalMarksPk(long totalMarksPk) {
+ this.totalMarksPk = totalMarksPk;
+ }
+
+ public void setSelectedMarksPk(long selectedMarksPk) {
+ this.selectedMarksPk = selectedMarksPk;
+ }
+
+ public long getSelectedMarks() {
+ return selectedMarks;
+ }
+
+ public void setSelectedMarks(long selectedMarks) {
+ this.selectedMarks = selectedMarks;
+ }
+
+ public long getTotalMarksPk() {
+ return totalMarksPk;
+ }
+
+ public long getSelectedMarksPk() {
+ return selectedMarksPk;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/metrics/NativeMetrics.java b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/NativeMetrics.java
new file mode 100644
index 000000000..449ef7e2a
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/NativeMetrics.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class NativeMetrics implements IMetrics {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NativeMetrics.class);
+
+ public List<MetricsData> metricsDataList;
+ public String metricsJson;
+
+ public NativeMetrics(String metricsJson) {
+ this.metricsJson = metricsJson;
+ this.metricsDataList = NativeMetrics.deserializeMetricsJson(this.metricsJson);
+ }
+
+ public void setFinalOutputMetrics(long outputRowCount, long outputVectorCount) {
+ if (CollectionUtils.isNotEmpty(this.metricsDataList)) {
+ int listSize = this.metricsDataList.size();
+ this.metricsDataList.get(listSize - 1).outputVectors = outputVectorCount;
+ this.metricsDataList.get(listSize - 1).outputRows = outputRowCount;
+
+ // skip 'kFetch' and 'kSort'
+ if (listSize > 2
+ && this.metricsDataList.get(listSize - 1).name.equals("kFetch")
+ && this.metricsDataList.get(listSize - 2).name.equals("kSort")) {
+ this.metricsDataList.get(listSize - 3).outputVectors = outputVectorCount;
+ this.metricsDataList.get(listSize - 3).outputRows = outputRowCount;
+ }
+ }
+ }
+
+ /** Deserialize metrics json string to MetricsData */
+ public static List<MetricsData> deserializeMetricsJson(String metricsJson) {
+ if (metricsJson != null && !metricsJson.isEmpty()) {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ List<MetricsData> metricsDataList =
+ mapper.readValue(metricsJson, new TypeReference<List<MetricsData>>() {});
+ Collections.reverse(metricsDataList);
+ return metricsDataList;
+ } catch (Exception e) {
+ LOG.error("Deserialize metrics json string error:", e);
+ return new ArrayList<MetricsData>();
+ }
+ }
+ return new ArrayList<MetricsData>();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
new file mode 100644
index 000000000..8dcb0ef74
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics;
+
+import org.apache.gluten.substrait.AggregationParams;
+import org.apache.gluten.substrait.JoinParams;
+
+import java.util.List;
+
+public class OperatorMetrics implements IOperatorMetrics {
+
+ public List<MetricsData> metricsList;
+ public JoinParams joinParams;
+ public AggregationParams aggParams;
+
+ /** Create an instance for operator metrics. */
+ public OperatorMetrics(
+ List<MetricsData> metricsList, JoinParams joinParams, AggregationParams aggParams) {
+ this.metricsList = metricsList;
+ this.aggParams = aggParams;
+ this.joinParams = joinParams;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java b/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java
new file mode 100644
index 000000000..23acaeec9
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils;
+
+import org.apache.gluten.exception.GlutenException;
+
+import org.apache.spark.SparkEnv;
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig;
+
+/**
+ * An object that generates IDs. This is broken into a separate class in case we ever want to
+ * support multiple worker threads per process. Refer to twitter-archive Snowflake.
+ */
+public class SnowflakeIdWorker {
+
+ // ==============================Singleton=====================================
+ private static volatile SnowflakeIdWorker INSTANCE;
+ // ==============================Fields===========================================
+ private final long twepoch = 1640966400L;
+ private final long workerIdBits = 6L;
+ private final long maxWorkerId = -1L ^ (-1L << workerIdBits);
+ private final long sequenceBits = 16L;
+ private final long workerIdShift = sequenceBits;
+ private final long timestampLeftShift = sequenceBits + workerIdBits;
+ private final long sequenceMask = -1L ^ (-1L << sequenceBits);
+ private long workerId;
+ private long sequence = 0L;
+ private long lastTimestamp = -1L;
+
+ public SnowflakeIdWorker(long workerId) {
+ if (workerId > maxWorkerId || workerId < 0) {
+ throw new IllegalArgumentException(
+ String.format("worker Id can't be greater than %d or less than 0", maxWorkerId));
+ }
+ this.workerId = workerId;
+ }
+
+ // ==============================Constructors=====================================
+
+ public static SnowflakeIdWorker getInstance() {
+ if (INSTANCE == null) {
+ synchronized (SnowflakeIdWorker.class) {
+ if (INSTANCE == null) {
+ if (!SparkEnv.get().conf().contains(ClickHouseConfig.CLICKHOUSE_WORKER_ID())) {
+ throw new IllegalArgumentException(
+ "Please set an unique value to " + ClickHouseConfig.CLICKHOUSE_WORKER_ID());
+ }
+ INSTANCE =
+ new SnowflakeIdWorker(
+ SparkEnv.get().conf().getLong(ClickHouseConfig.CLICKHOUSE_WORKER_ID(), 0));
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ // ==============================Methods==========================================
+ public synchronized long nextId() {
+ long timestamp = timeGen();
+
+ if (timestamp < lastTimestamp) {
+ throw new GlutenException(
+ String.format(
+ "Clock moved backwards. Refusing to generate id for %d milliseconds",
+ lastTimestamp - timestamp));
+ }
+
+ if (lastTimestamp == timestamp) {
+ sequence = (sequence + 1) & sequenceMask;
+ if (sequence == 0) {
+ timestamp = tilNextMillis(lastTimestamp);
+ }
+ } else {
+ sequence = 0L;
+ }
+
+ lastTimestamp = timestamp;
+
+ return ((timestamp - twepoch) << timestampLeftShift) //
+ | (workerId << workerIdShift) //
+ | sequence;
+ }
+
+ protected long tilNextMillis(long lastTimestamp) {
+ long timestamp = timeGen();
+ while (timestamp <= lastTimestamp) {
+ timestamp = timeGen();
+ }
+ return timestamp;
+ }
+
+ protected long timeGen() {
+ return System.currentTimeMillis() / 1000L;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java
new file mode 100644
index 000000000..5698caf02
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.metrics.IMetrics;
+
+import org.apache.spark.sql.execution.utils.CHExecUtil;
+import org.apache.spark.sql.vectorized.ColumnVector;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+import java.io.IOException;
+
+public class BatchIterator extends GeneralOutIterator {
+ private final long handle;
+
+ public BatchIterator(long handle) {
+ super();
+ this.handle = handle;
+ }
+
+ @Override
+ public String getId() {
+ // Using native handle as identifier
+ return String.valueOf(handle);
+ }
+
+ private native boolean nativeHasNext(long nativeHandle);
+
+ private native byte[] nativeNext(long nativeHandle);
+
+ private native long nativeCHNext(long nativeHandle);
+
+ private native void nativeClose(long nativeHandle);
+
+ private native IMetrics nativeFetchMetrics(long nativeHandle);
+
+ @Override
+ public boolean hasNextInternal() throws IOException {
+ return nativeHasNext(handle);
+ }
+
+ @Override
+ public ColumnarBatch nextInternal() throws IOException {
+ long block = nativeCHNext(handle);
+ CHNativeBlock nativeBlock = new CHNativeBlock(block);
+ int cols = nativeBlock.numColumns();
+ ColumnVector[] columnVectors = new ColumnVector[cols];
+ for (int i = 0; i < cols; i++) {
+ columnVectors[i] =
+ new CHColumnVector(
+ CHExecUtil.inferSparkDataType(nativeBlock.getTypeByPosition(i)), block, i);
+ }
+ return new ColumnarBatch(columnVectors, nativeBlock.numRows());
+ }
+
+ @Override
+ public IMetrics getMetricsInternal() throws IOException, ClassNotFoundException {
+ return nativeFetchMetrics(handle);
+ }
+
+ @Override
+ public void closeInternal() {
+ nativeClose(handle);
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockOutputStream.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockOutputStream.java
new file mode 100644
index 000000000..40e2c2c56
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockOutputStream.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.spark.sql.execution.metric.SQLMetric;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+import org.apache.spark.storage.CHShuffleWriteStreamFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class BlockOutputStream implements Closeable {
+ private final long instance;
+ private final OutputStream outputStream;
+
+ private final SQLMetric dataSize;
+
+ private boolean isClosed = false;
+
+ public BlockOutputStream(
+ OutputStream outputStream,
+ byte[] buffer,
+ SQLMetric dataSize,
+ boolean compressionEnable,
+ String defaultCompressionCodec,
+ int bufferSize) {
+ OutputStream unwrapOutputStream =
+ CHShuffleWriteStreamFactory.unwrapSparkCompressionOutputStream(
+ outputStream, compressionEnable);
+ if (unwrapOutputStream != null) {
+ this.outputStream = unwrapOutputStream;
+ } else {
+ this.outputStream = outputStream;
+ compressionEnable = false;
+ }
+ this.instance =
+ nativeCreate(
+ this.outputStream, buffer, defaultCompressionCodec, compressionEnable, bufferSize);
+ this.dataSize = dataSize;
+ }
+
+ private native long nativeCreate(
+ OutputStream outputStream,
+ byte[] buffer,
+ String defaultCompressionCodec,
+ boolean compressionEnable,
+ int bufferSize);
+
+ private native long nativeClose(long instance);
+
+ private native void nativeWrite(long instance, long block);
+
+ private native void nativeFlush(long instance);
+
+ public void write(ColumnarBatch cb) {
+ CHNativeBlock block = CHNativeBlock.fromColumnarBatch(cb);
+ dataSize.add(block.totalBytes());
+ nativeWrite(instance, block.blockAddress());
+ }
+
+ public void flush() throws IOException {
+ nativeFlush(instance);
+ this.outputStream.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!isClosed) {
+ nativeClose(instance);
+ this.outputStream.flush();
+ this.outputStream.close();
+ isClosed = true;
+ }
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ // FIXME: finalize
+ close();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockSplitIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockSplitIterator.java
new file mode 100644
index 000000000..407a0fc7a
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BlockSplitIterator.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+public class BlockSplitIterator implements Iterator<ColumnarBatch>, AutoCloseable {
+
+ private long instance = 0;
+
+ public BlockSplitIterator(Iterator<Long> in, IteratorOptions options) {
+ this.instance =
+ nativeCreate(
+ new IteratorWrapper(in),
+ options.getName(),
+ options.getExpr(),
+ options.getRequiredFields(),
+ options.getPartitionNum(),
+ options.getBufferSize(),
+ options.getHashAlgorithm());
+ }
+
+ private native long nativeCreate(
+ IteratorWrapper in,
+ String name,
+ String expr,
+ String schema,
+ int partitionNum,
+ int bufferSize,
+ String hashAlgorithm);
+
+ private native void nativeClose(long instance);
+
+ private native boolean nativeHasNext(long instance);
+
+ @Override
+ public boolean hasNext() {
+ return nativeHasNext(instance);
+ }
+
+ private native long nativeNext(long instance);
+
+ @Override
+ public ColumnarBatch next() {
+ CHNativeBlock block = new CHNativeBlock(nativeNext(instance));
+ return block.toColumnarBatch();
+ }
+
+ private native int nativeNextPartitionId(long instance);
+
+ public int nextPartitionId() {
+ return nativeNextPartitionId(instance);
+ }
+
+ @Override
+ public void close() throws Exception {
+ nativeClose(instance);
+ }
+
+ public static class IteratorOptions implements Serializable {
+ private static final long serialVersionUID = -1L;
+ private int partitionNum;
+ private String name;
+ private int bufferSize;
+ private String expr;
+ private String requiredFields;
+
+ private String hashAlgorithm;
+
+ public int getPartitionNum() {
+ return partitionNum;
+ }
+
+ public void setPartitionNum(int partitionNum) {
+ this.partitionNum = partitionNum;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public void setBufferSize(int bufferSize) {
+ this.bufferSize = bufferSize;
+ }
+
+ public String getExpr() {
+ return expr;
+ }
+
+ public void setExpr(String expr) {
+ this.expr = expr;
+ }
+
+ public String getRequiredFields() {
+ return requiredFields;
+ }
+
+ public void setRequiredFields(String requiredFields) {
+ this.requiredFields = requiredFields;
+ }
+
+ public String getHashAlgorithm() {
+ return hashAlgorithm;
+ }
+
+ public void setHashAlgorithm(String hashAlgorithm) {
+ this.hashAlgorithm = hashAlgorithm;
+ }
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockConverterJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockConverterJniWrapper.java
new file mode 100644
index 000000000..836e954f1
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockConverterJniWrapper.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.execution.SparkRowIterator;
+import org.apache.gluten.row.SparkRowInfo;
+
+public class CHBlockConverterJniWrapper {
+
+ private CHBlockConverterJniWrapper() {
+ // utility class
+ }
+
+ // for ch columnar -> spark row
+ public static native SparkRowInfo convertColumnarToRow(long blockAddress, int[] masks);
+
+ // for ch columnar -> spark row
+ public static native void freeMemory(long address, long size);
+
+ // for spark row -> ch columnar
+ public static native long convertSparkRowsToCHColumn(
+ SparkRowIterator iter, String[] names, byte[][] types);
+
+ // for spark row -> ch columnar
+ public static native void freeBlock(long blockAddress);
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockWriterJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockWriterJniWrapper.java
new file mode 100644
index 000000000..c460be84f
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHBlockWriterJniWrapper.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+public class CHBlockWriterJniWrapper {
+ private long instance = 0;
+
+ private native long nativeCreateInstance();
+
+ private native void nativeWrite(long instance, long block);
+
+ private native int nativeResultSize(long instance);
+
+ private native void nativeCollect(long instance, byte[] data);
+
+ private native void nativeClose(long instance);
+
+ public void write(ColumnarBatch columnarBatch) {
+ if (instance == 0) {
+ instance = nativeCreateInstance();
+ }
+ nativeWrite(instance, CHNativeBlock.fromColumnarBatch(columnarBatch).blockAddress());
+ }
+
+ public byte[] collectAsByteArray() {
+ if (instance == 0L) {
+ return new byte[0];
+ }
+ byte[] result = new byte[nativeResultSize(instance)];
+ nativeCollect(instance, result);
+ nativeClose(instance);
+ instance = 0;
+ return result;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHColumnVector.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHColumnVector.java
new file mode 100644
index 000000000..9cbb24282
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHColumnVector.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.vectorized.ColumnVector;
+import org.apache.spark.sql.vectorized.ColumnarArray;
+import org.apache.spark.sql.vectorized.ColumnarMap;
+import org.apache.spark.unsafe.types.UTF8String;
+
+public class CHColumnVector extends ColumnVector {
+ private final int columnPosition;
+ private long blockAddress;
+
+ public CHColumnVector(DataType type, long blockAddress, int columnPosition) {
+ super(type);
+ this.blockAddress = blockAddress;
+ this.columnPosition = columnPosition;
+ }
+
+ public long getBlockAddress() {
+ return blockAddress;
+ }
+
+ @Override
+ public void close() {
+ // blockAddress = 0;
+ }
+
+ private native boolean nativeHasNull(long blockAddress, int columnPosition);
+
+ @Override
+ public boolean hasNull() {
+ return nativeHasNull(blockAddress, columnPosition);
+ }
+
+ private native int nativeNumNulls(long blockAddress, int columnPosition);
+
+ @Override
+ public int numNulls() {
+ return nativeNumNulls(blockAddress, columnPosition);
+ }
+
+ private native boolean nativeIsNullAt(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public boolean isNullAt(int rowId) {
+ return nativeIsNullAt(rowId, blockAddress, columnPosition);
+ }
+
+ private native boolean nativeGetBoolean(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public boolean getBoolean(int rowId) {
+ return nativeGetBoolean(rowId, blockAddress, columnPosition);
+ }
+
+ private native byte nativeGetByte(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public byte getByte(int rowId) {
+ return nativeGetByte(rowId, blockAddress, columnPosition);
+ }
+
+ private native short nativeGetShort(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public short getShort(int rowId) {
+ return nativeGetShort(rowId, blockAddress, columnPosition);
+ }
+
+ private native int nativeGetInt(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public int getInt(int rowId) {
+ return nativeGetInt(rowId, blockAddress, columnPosition);
+ }
+
+ private native long nativeGetLong(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public long getLong(int rowId) {
+ return nativeGetLong(rowId, blockAddress, columnPosition);
+ }
+
+ private native float nativeGetFloat(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public float getFloat(int rowId) {
+ return nativeGetFloat(rowId, blockAddress, columnPosition);
+ }
+
+ private native double nativeGetDouble(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public double getDouble(int rowId) {
+ return nativeGetDouble(rowId, blockAddress, columnPosition);
+ }
+
+ @Override
+ public ColumnarArray getArray(int rowId) {
+ return null;
+ }
+
+ @Override
+ public ColumnarMap getMap(int ordinal) {
+ return null;
+ }
+
+ @Override
+ public Decimal getDecimal(int rowId, int precision, int scale) {
+ return null;
+ }
+
+ private native String nativeGetString(int rowId, long blockAddress, int columnPosition);
+
+ @Override
+ public UTF8String getUTF8String(int rowId) {
+ return UTF8String.fromString(nativeGetString(rowId, blockAddress, columnPosition));
+ }
+
+ @Override
+ public byte[] getBinary(int rowId) {
+ return new byte[0];
+ }
+
+ @Override
+ public ColumnVector getChild(int ordinal) {
+ return null;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeBlock.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeBlock.java
new file mode 100644
index 000000000..f84327e7d
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeBlock.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.exception.GlutenException;
+
+import org.apache.spark.sql.execution.utils.CHExecUtil;
+import org.apache.spark.sql.vectorized.ColumnVector;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+public class CHNativeBlock {
+ private long blockAddress;
+
+ public CHNativeBlock(long blockAddress) {
+ this.blockAddress = blockAddress;
+ }
+
+ public static CHNativeBlock fromColumnarBatch(ColumnarBatch batch) {
+ if (batch.numCols() == 0 || !(batch.column(0) instanceof CHColumnVector)) {
+ throw new GlutenException(
+ "Unexpected ColumnarBatch: "
+ + (batch.numCols() == 0
+ ? "0 column"
+ : "expected CHColumnVector, but " + batch.column(0).getClass()));
+ }
+ CHColumnVector columnVector = (CHColumnVector) batch.column(0);
+ return new CHNativeBlock(columnVector.getBlockAddress());
+ }
+
+ private native int nativeNumRows(long blockAddress);
+
+ public int numRows() {
+ return nativeNumRows(blockAddress);
+ }
+
+ public long blockAddress() {
+ return blockAddress;
+ }
+
+ private native int nativeNumColumns(long blockAddress);
+
+ public int numColumns() {
+ return nativeNumColumns(blockAddress);
+ }
+
+ private native byte[] nativeColumnType(long blockAddress, int position);
+
+ public byte[] getTypeByPosition(int position) {
+ return nativeColumnType(blockAddress, position);
+ }
+
+ private native long nativeTotalBytes(long blockAddress);
+
+ public long totalBytes() {
+ return nativeTotalBytes(blockAddress);
+ }
+
+ public native void nativeClose(long blockAddress);
+
+ public void close() {
+ if (blockAddress != 0) {
+ nativeClose(blockAddress);
+ blockAddress = 0;
+ }
+ }
+
+ public static void closeFromColumnarBatch(ColumnarBatch cb) {
+ if (cb != null) {
+ if (cb.numCols() > 0) {
+ CHColumnVector col = (CHColumnVector) cb.column(0);
+ CHNativeBlock block = new CHNativeBlock(col.getBlockAddress());
+ block.close();
+ }
+ cb.close();
+ }
+ }
+
+ public ColumnarBatch toColumnarBatch() {
+ ColumnVector[] vectors = new ColumnVector[numColumns()];
+ for (int i = 0; i < numColumns(); i++) {
+ vectors[i] =
+ new CHColumnVector(CHExecUtil.inferSparkDataType(getTypeByPosition(i)), blockAddress, i);
+ }
+ int numRows = 0;
+ if (numColumns() != 0) {
+ numRows = numRows();
+ }
+ return new ColumnarBatch(vectors, numRows);
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java
new file mode 100644
index 000000000..0d307d231
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.GlutenConfig;
+import org.apache.gluten.backendsapi.BackendsApiManager;
+import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators;
+import org.apache.gluten.substrait.expression.ExpressionBuilder;
+import org.apache.gluten.substrait.expression.StringMapNode;
+import org.apache.gluten.substrait.extensions.AdvancedExtensionNode;
+import org.apache.gluten.substrait.extensions.ExtensionBuilder;
+import org.apache.gluten.substrait.plan.PlanBuilder;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.internal.SQLConf;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+
+public class CHNativeExpressionEvaluator {
+ private final ExpressionEvaluatorJniWrapper jniWrapper;
+
+ public CHNativeExpressionEvaluator() {
+ jniWrapper = new ExpressionEvaluatorJniWrapper();
+ }
+
+ // Used to initialize the native computing.
+ public void initNative(SparkConf conf) {
+ Tuple2<String, String>[] all = conf.getAll();
+ Map<String, String> confMap =
+ Arrays.stream(all).collect(Collectors.toMap(Tuple2::_1, Tuple2::_2));
+ String prefix = BackendsApiManager.getSettings().getBackendConfigPrefix();
+ Map<String, String> nativeConfMap =
+ GlutenConfig.getNativeBackendConf(prefix, JavaConverters.mapAsScalaMap(confMap));
+
+ // Get the customer config from SparkConf for each backend
+ BackendsApiManager.getTransformerApiInstance().postProcessNativeConfig(nativeConfMap, prefix);
+
+ jniWrapper.nativeInitNative(buildNativeConf(nativeConfMap));
+ }
+
+ public void finalizeNative() {
+ jniWrapper.nativeFinalizeNative();
+ }
+
+ // Used to validate the Substrait plan in native compute engine.
+ public boolean doValidate(byte[] subPlan) {
+ return jniWrapper.nativeDoValidate(subPlan);
+ }
+
+ private byte[] buildNativeConf(Map<String, String> confs) {
+ StringMapNode stringMapNode = ExpressionBuilder.makeStringMap(confs);
+ AdvancedExtensionNode extensionNode =
+ ExtensionBuilder.makeAdvancedExtension(
+ BackendsApiManager.getTransformerApiInstance()
+ .packPBMessage(stringMapNode.toProtobuf()));
+ return PlanBuilder.makePlan(extensionNode).toProtobuf().toByteArray();
+ }
+
+ private Map<String, String> getNativeBackendConf() {
+ return GlutenConfig.getNativeBackendConf(
+ BackendsApiManager.getSettings().getBackendConfigPrefix(), SQLConf.get().getAllConfs());
+ }
+
+ // Used by WholeStageTransform to create the native computing pipeline and
+ // return a columnar result iterator.
+ public GeneralOutIterator createKernelWithBatchIterator(
+ byte[] wsPlan,
+ byte[][] splitInfo,
+ List<GeneralInIterator> iterList,
+ boolean materializeInput) {
+ long allocId = CHNativeMemoryAllocators.contextInstance().getNativeInstanceId();
+ long handle =
+ jniWrapper.nativeCreateKernelWithIterator(
+ allocId,
+ wsPlan,
+ splitInfo,
+ iterList.toArray(new GeneralInIterator[0]),
+ buildNativeConf(getNativeBackendConf()),
+ materializeInput);
+ return createOutIterator(handle);
+ }
+
+ // Only for UT.
+ public GeneralOutIterator createKernelWithBatchIterator(
+ long allocId, byte[] wsPlan, byte[][] splitInfo, List<GeneralInIterator> iterList) {
+ long handle =
+ jniWrapper.nativeCreateKernelWithIterator(
+ allocId,
+ wsPlan,
+ splitInfo,
+ iterList.toArray(new GeneralInIterator[0]),
+ buildNativeConf(getNativeBackendConf()),
+ false);
+ return createOutIterator(handle);
+ }
+
+ private GeneralOutIterator createOutIterator(long nativeHandle) {
+ return new BatchIterator(nativeHandle);
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java
new file mode 100644
index 000000000..800ee7d19
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import java.io.IOException;
+
+public class CHShuffleSplitterJniWrapper {
+ public CHShuffleSplitterJniWrapper() {}
+
+ public long make(
+ NativePartitioning part,
+ int shuffleId,
+ long mapId,
+ int bufferSize,
+ String codec,
+ String dataFile,
+ String localDirs,
+ int subDirsPerLocalDir,
+ boolean preferSpill,
+ long spillThreshold,
+ String hashAlgorithm,
+ boolean throwIfMemoryExceed,
+ boolean flushBlockBufferBeforeEvict) {
+ return nativeMake(
+ part.getShortName(),
+ part.getNumPartitions(),
+ part.getExprList(),
+ part.getRequiredFields(),
+ shuffleId,
+ mapId,
+ bufferSize,
+ codec,
+ dataFile,
+ localDirs,
+ subDirsPerLocalDir,
+ preferSpill,
+ spillThreshold,
+ hashAlgorithm,
+ throwIfMemoryExceed,
+ flushBlockBufferBeforeEvict);
+ }
+
+ public long makeForRSS(
+ NativePartitioning part,
+ int shuffleId,
+ long mapId,
+ int bufferSize,
+ String codec,
+ long spillThreshold,
+ String hashAlgorithm,
+ Object pusher,
+ boolean throwIfMemoryExceed,
+ boolean flushBlockBufferBeforeEvict) {
+ return nativeMakeForRSS(
+ part.getShortName(),
+ part.getNumPartitions(),
+ part.getExprList(),
+ part.getRequiredFields(),
+ shuffleId,
+ mapId,
+ bufferSize,
+ codec,
+ spillThreshold,
+ hashAlgorithm,
+ pusher,
+ throwIfMemoryExceed,
+ flushBlockBufferBeforeEvict);
+ }
+
+ public native long nativeMake(
+ String shortName,
+ int numPartitions,
+ byte[] exprList,
+ byte[] exprIndexList,
+ int shuffleId,
+ long mapId,
+ int bufferSize,
+ String codec,
+ String dataFile,
+ String localDirs,
+ int subDirsPerLocalDir,
+ boolean preferSpill,
+ long spillThreshold,
+ String hashAlgorithm,
+ boolean throwIfMemoryExceed,
+ boolean flushBlockBufferBeforeEvict);
+
+ public native long nativeMakeForRSS(
+ String shortName,
+ int numPartitions,
+ byte[] exprList,
+ byte[] exprIndexList,
+ int shuffleId,
+ long mapId,
+ int bufferSize,
+ String codec,
+ long spillThreshold,
+ String hashAlgorithm,
+ Object pusher,
+ boolean throwIfMemoryExceed,
+ boolean flushBlockBufferBeforeEvict);
+
+ public native void split(long splitterId, long block);
+
+ public native long evict(long splitterId);
+
+ public native CHSplitResult stop(long splitterId) throws IOException;
+
+ public native void close(long splitterId);
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHStreamReader.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHStreamReader.java
new file mode 100644
index 000000000..3151adfce
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHStreamReader.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings;
+
+import org.apache.spark.storage.CHShuffleReadStreamFactory;
+
+import java.io.InputStream;
+
+public class CHStreamReader implements AutoCloseable {
+ private final ShuffleInputStream inputStream;
+ private long nativeShuffleReader;
+
+ public CHStreamReader(
+ InputStream inputStream, boolean forceCompress, boolean isCustomizedShuffleCodec) {
+ this(CHShuffleReadStreamFactory.create(inputStream, forceCompress, isCustomizedShuffleCodec));
+ }
+
+ public CHStreamReader(ShuffleInputStream shuffleInputStream) {
+ inputStream = shuffleInputStream;
+ nativeShuffleReader =
+ createNativeShuffleReader(
+ this.inputStream,
+ inputStream.isCompressed(),
+ CHBackendSettings.maxShuffleReadRows(),
+ CHBackendSettings.maxShuffleReadBytes());
+ }
+
+ private static native long createNativeShuffleReader(
+ ShuffleInputStream inputStream,
+ boolean compressed,
+ long maxShuffleReadRows,
+ long maxShuffleReadBytes);
+
+ private native long nativeNext(long nativeShuffleReader);
+
+ public CHNativeBlock next() {
+ long block = nativeNext(nativeShuffleReader);
+ return new CHNativeBlock(block);
+ }
+
+ private native void nativeClose(long shuffleReader);
+
+ @Override
+ public void close() throws Exception {
+ // close input stream and release buffer
+ this.inputStream.close();
+ nativeClose(nativeShuffleReader);
+ nativeShuffleReader = 0L;
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java
new file mode 100644
index 000000000..0a008a04b
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+/**
+ * This class is implemented in JNI. This provides the Java interface to invoke functions in JNI.
+ * This file is used to generate the .h files required for jni. Avoid all external dependencies in
+ * this file.
+ */
+public class ExpressionEvaluatorJniWrapper {
+
+ /** Call initNative to initialize native computing. */
+ native void nativeInitNative(byte[] confAsPlan);
+
+ /** Call finalizeNative to finalize native computing. */
+ native void nativeFinalizeNative();
+
+ /**
+ * Validate the Substrait plan in native compute engine.
+ *
+ * @param subPlan the Substrait plan in binary format.
+ * @return whether the computing of this plan is supported in native.
+ */
+ native boolean nativeDoValidate(byte[] subPlan);
+
+ /**
+ * Create a native compute kernel and return a columnar result iterator.
+ *
+ * @param allocatorId allocator id
+ * @return iterator instance id
+ */
+ public native long nativeCreateKernelWithIterator(
+ long allocatorId,
+ byte[] wsPlan,
+ byte[][] splitInfo,
+ GeneralInIterator[] batchItr,
+ byte[] confArray,
+ boolean materializeInput);
+
+ /**
+ * Closes the projector referenced by nativeHandler.
+ *
+ * @param nativeHandler nativeHandler that needs to be closed
+ */
+ native void nativeClose(long nativeHandler);
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/IteratorWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/IteratorWrapper.java
new file mode 100644
index 000000000..2cf6ff50f
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/IteratorWrapper.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import java.util.Iterator;
+
+public class IteratorWrapper {
+ private final Iterator<Long> in;
+
+ public IteratorWrapper(Iterator<Long> in) {
+ this.in = in;
+ }
+
+ public boolean hasNext() {
+ return in.hasNext();
+ }
+
+ public long next() {
+ return in.next();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyFileSegmentShuffleInputStream.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyFileSegmentShuffleInputStream.java
new file mode 100644
index 000000000..3d9157a8e
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyFileSegmentShuffleInputStream.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.exception.GlutenException;
+
+import io.netty.util.internal.PlatformDependent;
+import org.apache.spark.network.util.LimitedInputStream;
+import org.apache.spark.storage.CHShuffleReadStreamFactory;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+public class LowCopyFileSegmentShuffleInputStream implements ShuffleInputStream {
+
+ private final InputStream in;
+ private final FileChannel channel;
+ private final boolean isCompressed;
+
+ private long bytesRead = 0L;
+ private long left;
+
+ public LowCopyFileSegmentShuffleInputStream(
+ InputStream in, LimitedInputStream limitedInputStream, boolean isCompressed) {
+ // to prevent underlying netty buffer from being collected by GC
+ this.in = in;
+ this.isCompressed = isCompressed;
+ final FileInputStream fin;
+ try {
+ left =
+ ((long) CHShuffleReadStreamFactory.FIELD_LimitedInputStream_left.get(limitedInputStream));
+ fin =
+ (FileInputStream)
+ CHShuffleReadStreamFactory.FIELD_FilterInputStream_in.get(limitedInputStream);
+ } catch (IllegalAccessException e) {
+ throw new GlutenException(e);
+ }
+ channel = fin.getChannel();
+ }
+
+ @Override
+ public long read(long destAddress, long maxReadSize) {
+ long bytesToRead = Math.min(left, maxReadSize);
+ int bytesToRead32 = Math.toIntExact(bytesToRead);
+ if (bytesToRead32 == 0) {
+ return 0;
+ }
+ ByteBuffer direct = PlatformDependent.directBuffer(destAddress, bytesToRead32);
+ try {
+ // read data from file channel to native address
+ int bytes = channel.read(direct);
+ if (bytes == -1) {
+ return 0;
+ }
+ bytesRead += bytes;
+ left -= bytes;
+ return bytes;
+ } catch (IOException e) {
+ throw new GlutenException(e);
+ }
+ }
+
+ @Override
+ public long pos() {
+ return bytesRead;
+ }
+
+ @Override
+ public boolean isCompressed() {
+ return this.isCompressed;
+ }
+
+ @Override
+ public void close() {
+ GlutenException.wrap(
+ () -> {
+ channel.close();
+ in.close();
+ return null;
+ });
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyNettyShuffleInputStream.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyNettyShuffleInputStream.java
new file mode 100644
index 000000000..0eb921579
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/LowCopyNettyShuffleInputStream.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.exception.GlutenException;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public class LowCopyNettyShuffleInputStream implements ShuffleInputStream {
+
+ private final InputStream in;
+ private final boolean isCompressed;
+
+ private final ByteBuf byteBuf;
+ private int readBytesCount = 0;
+
+ public LowCopyNettyShuffleInputStream(InputStream in, ByteBuf byteBuf, boolean isCompressed) {
+ // to prevent underlying netty buffer from being collected by GC
+ this.in = in;
+ this.byteBuf = byteBuf;
+ this.isCompressed = isCompressed;
+ }
+
+ @Override
+ public long read(long destAddress, long maxReadSize) {
+ long bytesToRead = Math.min(maxReadSize, this.byteBuf.readableBytes());
+ int bytesToRead32 = Math.toIntExact(bytesToRead);
+ if (bytesToRead32 == 0) {
+ return 0;
+ }
+ ByteBuffer direct = PlatformDependent.directBuffer(destAddress, bytesToRead32);
+ // read data directly from ByteBuf to native address
+ this.byteBuf.readBytes(direct);
+ readBytesCount += direct.position();
+ return direct.position();
+ }
+
+ @Override
+ public long pos() {
+ return readBytesCount;
+ }
+
+ @Override
+ public boolean isCompressed() {
+ return this.isCompressed;
+ }
+
+ @Override
+ public void close() {
+ GlutenException.wrap(
+ () -> {
+ in.close();
+ return null;
+ });
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/OnHeapCopyShuffleInputStream.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/OnHeapCopyShuffleInputStream.java
new file mode 100644
index 000000000..ea7a1e236
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/OnHeapCopyShuffleInputStream.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.exception.GlutenException;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.InputStream;
+
+public class OnHeapCopyShuffleInputStream implements ShuffleInputStream {
+
+ private InputStream in;
+ private final boolean isCompressed;
+ private long bytesRead = 0L;
+
+ private byte[] buffer;
+
+ public OnHeapCopyShuffleInputStream(InputStream in, boolean isCompressed) {
+ this.in = in;
+ this.isCompressed = isCompressed;
+ }
+
+ @Override
+ public long read(long destAddress, long maxReadSize) {
+ return GlutenException.wrap(
+ () -> {
+ int maxReadSize32 = Math.toIntExact(maxReadSize);
+ if (buffer == null || maxReadSize32 > buffer.length) {
+ this.buffer = new byte[maxReadSize32];
+ }
+ // The code conducts copy as long as 'in' wraps off-heap data,
+ // which is about to be moved to heap
+ int read = in.read(buffer, 0, maxReadSize32);
+ if (read == -1 || read == 0) {
+ return 0;
+ }
+ // The code conducts copy, from heap to off-heap
+ // memCopyFromHeap(buffer, destAddress, read);
+ PlatformDependent.copyMemory(buffer, 0, destAddress, read);
+ bytesRead += read;
+ return read;
+ });
+ }
+
+ @Override
+ public long pos() {
+ return bytesRead;
+ }
+
+ @Override
+ public boolean isCompressed() {
+ return this.isCompressed;
+ }
+
+ @Override
+ public void close() {
+ GlutenException.wrap(
+ () -> {
+ in.close();
+ in = null;
+ return null;
+ });
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ShuffleInputStream.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ShuffleInputStream.java
new file mode 100644
index 000000000..3f30cc7e4
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ShuffleInputStream.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+public interface ShuffleInputStream {
+
+ /**
+ * Read fixed size of data into an offheap address.
+ *
+ * @return the read bytes; 0 if end of stream.
+ */
+ long read(long destAddress, long maxReadSize);
+
+ boolean isCompressed();
+
+ /** Position of this stream. */
+ long pos();
+
+ /** Close and reclaim the resources. */
+ void close();
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/SimpleExpressionEval.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/SimpleExpressionEval.java
new file mode 100644
index 000000000..b09cccb45
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/SimpleExpressionEval.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.execution.ColumnarNativeIterator;
+import org.apache.gluten.substrait.plan.PlanNode;
+
+import io.substrait.proto.Plan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+public class SimpleExpressionEval implements AutoCloseable, Iterator<Long> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SimpleExpressionEval.class);
+
+ private final Long instance;
+
+ public SimpleExpressionEval(ColumnarNativeIterator blockStream, PlanNode planNode) {
+ Plan plan = planNode.toProtobuf();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("SimpleExpressionEval exec plan: %s", plan.toString()));
+ }
+ byte[] planData = plan.toByteArray();
+ instance = createNativeInstance(blockStream, planData);
+ }
+
+ private static native long createNativeInstance(ColumnarNativeIterator blockStream, byte[] plan);
+
+ private static native void nativeClose(long instance);
+
+ private static native boolean nativeHasNext(long instance);
+
+ private static native long nativeNext(long instance);
+
+ @Override
+ public boolean hasNext() {
+ return nativeHasNext(instance);
+ }
+
+ @Override
+ public Long next() {
+ return nativeNext(instance);
+ }
+
+ @Override
+ public void close() throws Exception {
+ nativeClose(instance);
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java
new file mode 100644
index 000000000..065be9de2
--- /dev/null
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.vectorized;
+
+import org.apache.gluten.execution.BroadCastHashJoinContext;
+import org.apache.gluten.expression.ConverterUtils;
+import org.apache.gluten.expression.ConverterUtils$;
+import org.apache.gluten.substrait.type.TypeNode;
+import org.apache.gluten.utils.SubstraitUtil;
+
+import io.substrait.proto.NamedStruct;
+import io.substrait.proto.Type;
+import org.apache.spark.sql.catalyst.expressions.Attribute;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import scala.collection.JavaConverters;
+
+public class StorageJoinBuilder {
+
+ public static native void nativeCleanBuildHashTable(String hashTableId, long hashTableData);
+
+ public static native long nativeCloneBuildHashTable(long hashTableData);
+
+ private static native long nativeBuild(
+ String buildHashTableId,
+ byte[] in,
+ long rowCount,
+ String joinKeys,
+ int joinType,
+ byte[] namedStruct);
+
+ private StorageJoinBuilder() {}
+
+ /** build storage join object */
+ public static long build(
+ byte[] batches,
+ long rowCount,
+ BroadCastHashJoinContext broadCastContext,
+ List<Expression> newBuildKeys,
+ List<Attribute> newOutput) {
+ ConverterUtils$ converter = ConverterUtils$.MODULE$;
+ List<Expression> keys;
+ List<Attribute> output;
+ if (newBuildKeys.isEmpty()) {
+ keys = JavaConverters.<Expression>seqAsJavaList(broadCastContext.buildSideJoinKeys());
+ output = JavaConverters.<Attribute>seqAsJavaList(broadCastContext.buildSideStructure());
+ } else {
+ keys = newBuildKeys;
+ output = newOutput;
+ }
+ String joinKey =
+ keys.stream()
+ .map(
+ (Expression key) -> {
+ Attribute attr = converter.getAttrFromExpr(key);
+ return converter.genColumnNameWithExprId(attr);
+ })
+ .collect(Collectors.joining(","));
+ return nativeBuild(
+ broadCastContext.buildHashTableId(),
+ batches,
+ rowCount,
+ joinKey,
+ SubstraitUtil.toSubstrait(broadCastContext.joinType()).ordinal(),
+ toNameStruct(output).toByteArray());
+ }
+
+ /** create table named struct */
+ private static NamedStruct toNameStruct(List<Attribute> output) {
+ List<TypeNode> typeList = ConverterUtils.collectAttributeTypeNodes(output);
+ List<String> nameList = ConverterUtils.collectAttributeNamesWithExprId(output);
+ Type.Struct.Builder structBuilder = Type.Struct.newBuilder();
+ for (TypeNode typeNode : typeList) {
+ structBuilder.addTypes(typeNode.toProtobuf());
+ }
+ NamedStruct.Builder nStructBuilder = NamedStruct.newBuilder();
+ nStructBuilder.setStruct(structBuilder.build());
+ for (String name : nameList) {
+ nStructBuilder.addNames(name);
+ }
+ return nStructBuilder.build();
+ }
+}
diff --git a/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleReadStreamFactory.java b/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleReadStreamFactory.java
index f60bafbc4..acc68e173 100644
--- a/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleReadStreamFactory.java
+++ b/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleReadStreamFactory.java
@@ -16,11 +16,11 @@
*/
package org.apache.spark.storage;
-import io.glutenproject.exception.GlutenException;
-import io.glutenproject.vectorized.LowCopyFileSegmentShuffleInputStream;
-import io.glutenproject.vectorized.LowCopyNettyShuffleInputStream;
-import io.glutenproject.vectorized.OnHeapCopyShuffleInputStream;
-import io.glutenproject.vectorized.ShuffleInputStream;
+import org.apache.gluten.exception.GlutenException;
+import org.apache.gluten.vectorized.LowCopyFileSegmentShuffleInputStream;
+import org.apache.gluten.vectorized.LowCopyNettyShuffleInputStream;
+import org.apache.gluten.vectorized.OnHeapCopyShuffleInputStream;
+import org.apache.gluten.vectorized.ShuffleInputStream;
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
import com.ning.compress.lzf.LZFInputStream;
diff --git a/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleWriteStreamFactory.java b/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleWriteStreamFactory.java
index b8816bd46..edec10791 100644
--- a/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleWriteStreamFactory.java
+++ b/backends-clickhouse/src/main/java/org/apache/spark/storage/CHShuffleWriteStreamFactory.java
@@ -16,7 +16,7 @@
*/
package org.apache.spark.storage;
-import io.glutenproject.exception.GlutenException;
+import org.apache.gluten.exception.GlutenException;
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
import com.ning.compress.lzf.LZFOutputStream;
diff --git a/backends-clickhouse/src/main/resources/META-INF/services/io.glutenproject.backendsapi.Backend b/backends-clickhouse/src/main/resources/META-INF/services/io.glutenproject.backendsapi.Backend
deleted file mode 100644
index 9c55de68c..000000000
--- a/backends-clickhouse/src/main/resources/META-INF/services/io.glutenproject.backendsapi.Backend
+++ /dev/null
@@ -1 +0,0 @@
-io.glutenproject.backendsapi.clickhouse.CHBackend
diff --git a/backends-clickhouse/src/main/resources/META-INF/services/org.apache.gluten.backendsapi.Backend b/backends-clickhouse/src/main/resources/META-INF/services/org.apache.gluten.backendsapi.Backend
new file mode 100644
index 000000000..bcd3cb1c0
--- /dev/null
+++ b/backends-clickhouse/src/main/resources/META-INF/services/org.apache.gluten.backendsapi.Backend
@@ -0,0 +1 @@
+org.apache.gluten.backendsapi.clickhouse.CHBackend
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBackend.scala
deleted file mode 100644
index a7c5c9980..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBackend.scala
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.{CH_BRANCH, CH_COMMIT, GlutenConfig}
-import io.glutenproject.backendsapi._
-import io.glutenproject.expression.WindowFunctionsBuilder
-import io.glutenproject.extension.ValidationResult
-import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat
-import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat._
-
-import org.apache.spark.SparkEnv
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.catalog.BucketSpec
-import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, DenseRank, Lag, Lead, NamedExpression, Rank, RowNumber}
-import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning}
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.aggregate.HashAggregateExec
-import org.apache.spark.sql.execution.datasources.FileFormat
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{ArrayType, MapType, StructField, StructType}
-
-import java.util.Locale
-
-import scala.util.control.Breaks.{break, breakable}
-
-class CHBackend extends Backend {
- override def name(): String = CHBackend.BACKEND_NAME
- override def buildInfo(): BackendBuildInfo =
- BackendBuildInfo("ClickHouse", CH_BRANCH, CH_COMMIT, "UNKNOWN")
- override def iteratorApi(): IteratorApi = new CHIteratorApi
- override def sparkPlanExecApi(): SparkPlanExecApi = new CHSparkPlanExecApi
- override def transformerApi(): TransformerApi = new CHTransformerApi
- override def validatorApi(): ValidatorApi = new CHValidatorApi
- override def metricsApi(): MetricsApi = new CHMetricsApi
- override def listenerApi(): ListenerApi = new CHListenerApi
- override def broadcastApi(): BroadcastApi = new CHBroadcastApi
- override def settings(): BackendSettingsApi = CHBackendSettings
-}
-
-object CHBackend {
- val BACKEND_NAME = "ch"
-}
-
-object CHBackendSettings extends BackendSettingsApi with Logging {
-
- private val GLUTEN_CLICKHOUSE_SEP_SCAN_RDD = "spark.gluten.sql.columnar.separate.scan.rdd.for.ch"
- private val GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT = "false"
-
- // experimental: when the files count per partition exceeds this threshold,
- // it will put the files into one partition.
- val GLUTEN_CLICKHOUSE_FILES_PER_PARTITION_THRESHOLD: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".files.per.partition.threshold"
- val GLUTEN_CLICKHOUSE_FILES_PER_PARTITION_THRESHOLD_DEFAULT = "-1"
-
- private val GLUTEN_CLICKHOUSE_CUSTOMIZED_SHUFFLE_CODEC_ENABLE: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".customized.shuffle.codec.enable"
- private val GLUTEN_CLICKHOUSE_CUSTOMIZED_SHUFFLE_CODEC_ENABLE_DEFAULT = false
- lazy val useCustomizedShuffleCodec: Boolean = SparkEnv.get.conf.getBoolean(
- CHBackendSettings.GLUTEN_CLICKHOUSE_CUSTOMIZED_SHUFFLE_CODEC_ENABLE,
- CHBackendSettings.GLUTEN_CLICKHOUSE_CUSTOMIZED_SHUFFLE_CODEC_ENABLE_DEFAULT
- )
-
- private val GLUTEN_CLICKHOUSE_CUSTOMIZED_BUFFER_SIZE: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".customized.buffer.size"
- private val GLUTEN_CLICKHOUSE_CUSTOMIZED_BUFFER_SIZE_DEFAULT = 4096
- lazy val customizeBufferSize: Int = SparkEnv.get.conf.getInt(
- CHBackendSettings.GLUTEN_CLICKHOUSE_CUSTOMIZED_BUFFER_SIZE,
- CHBackendSettings.GLUTEN_CLICKHOUSE_CUSTOMIZED_BUFFER_SIZE_DEFAULT
- )
-
- val GLUTEN_CLICKHOUSE_BROADCAST_CACHE_EXPIRED_TIME: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".broadcast.cache.expired.time"
- // unit: SECONDS, default 1 day
- val GLUTEN_CLICKHOUSE_BROADCAST_CACHE_EXPIRED_TIME_DEFAULT: Int = 86400
-
- private val GLUTEN_CLICKHOUSE_SHUFFLE_SUPPORTED_CODEC: Set[String] = Set("lz4", "zstd", "snappy")
-
- // The algorithm for hash partition of the shuffle
- private val GLUTEN_CLICKHOUSE_SHUFFLE_HASH_ALGORITHM: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".shuffle.hash.algorithm"
- // valid values are: cityHash64 or sparkMurmurHash3_32
- private val GLUTEN_CLICKHOUSE_SHUFFLE_HASH_ALGORITHM_DEFAULT = "sparkMurmurHash3_32"
- def shuffleHashAlgorithm: String = {
- val algorithm = SparkEnv.get.conf.get(
- CHBackendSettings.GLUTEN_CLICKHOUSE_SHUFFLE_HASH_ALGORITHM,
- CHBackendSettings.GLUTEN_CLICKHOUSE_SHUFFLE_HASH_ALGORITHM_DEFAULT
- )
- if (!algorithm.equals("cityHash64") && !algorithm.equals("sparkMurmurHash3_32")) {
- CHBackendSettings.GLUTEN_CLICKHOUSE_SHUFFLE_HASH_ALGORITHM_DEFAULT
- } else {
- algorithm
- }
- }
-
- val GLUTEN_CLICKHOUSE_AFFINITY_MODE: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME + ".affinity.mode"
- val SOFT: String = "soft"
- val FORCE: String = "force"
- private val GLUTEN_CLICKHOUSE_AFFINITY_MODE_DEFAULT = SOFT
-
- val GLUTEN_MAX_BLOCK_SIZE: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".runtime_settings.max_block_size"
- // Same as default value in clickhouse
- val GLUTEN_MAX_BLOCK_SIZE_DEFAULT = 65409
- val GLUTEN_MAX_SHUFFLE_READ_BYTES: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME +
- ".runtime_config.max_source_concatenate_bytes"
- val GLUTEN_MAX_SHUFFLE_READ_BYTES_DEFAULT = -1
-
- def affinityMode: String = {
- SparkEnv.get.conf
- .get(
- CHBackendSettings.GLUTEN_CLICKHOUSE_AFFINITY_MODE,
- CHBackendSettings.GLUTEN_CLICKHOUSE_AFFINITY_MODE_DEFAULT
- )
- .toLowerCase(Locale.getDefault)
- }
-
- override def supportFileFormatRead(
- format: ReadFileFormat,
- fields: Array[StructField],
- partTable: Boolean,
- paths: Seq[String]): ValidationResult = {
-
- def validateFilePath: Boolean = {
- // Fallback to vanilla spark when the input path
- // does not contain the partition info.
- if (partTable && !paths.forall(_.contains("="))) {
- return false
- }
- true
- }
-
- // Validate if all types are supported.
- def hasComplexType: Boolean = {
- // Collect unsupported types.
- val unsupportedDataTypes = fields.map(_.dataType).collect {
- case _: MapType => "MapType"
- case _: StructType => "StructType"
- case _: ArrayType => "ArrayType"
- }
- for (unsupportedDataType <- unsupportedDataTypes) {
- // scalastyle:off println
- println(
- s"Validation failed for ${this.getClass.toString}" +
- s" due to: data type $unsupportedDataType. in file schema. ")
- // scalastyle:on println
- }
- !unsupportedDataTypes.isEmpty
- }
- format match {
- case ParquetReadFormat =>
- if (validateFilePath) {
- ValidationResult.ok
- } else {
- ValidationResult.notOk("Validate file path failed.")
- }
- case OrcReadFormat => ValidationResult.ok
- case MergeTreeReadFormat => ValidationResult.ok
- case TextReadFormat =>
- if (!hasComplexType) {
- ValidationResult.ok
- } else {
- ValidationResult.notOk("Has complex type.")
- }
- case JsonReadFormat => ValidationResult.ok
- case _ => ValidationResult.notOk(s"Unsupported file format $format")
- }
- }
-
- override def utilizeShuffledHashJoinHint(): Boolean = true
- override def supportShuffleWithProject(
- outputPartitioning: Partitioning,
- child: SparkPlan): Boolean = {
- child match {
- case hash: HashAggregateExec =>
- if (hash.aggregateExpressions.isEmpty) {
- true
- } else {
- outputPartitioning match {
- case hashPartitioning: HashPartitioning =>
- hashPartitioning.expressions.exists(x => !x.isInstanceOf[AttributeReference])
- case _ =>
- false
- }
- }
- case _ =>
- true
- }
- }
-
- override def supportSortExec(): Boolean = {
- GlutenConfig.getConf.enableColumnarSort
- }
-
- override def supportSortMergeJoinExec(): Boolean = {
- false
- }
-
- override def supportWindowExec(windowFunctions: Seq[NamedExpression]): Boolean = {
- var allSupported = true
- breakable {
- windowFunctions.foreach(
- func => {
- val aliasExpr = func.asInstanceOf[Alias]
- val wExpression = WindowFunctionsBuilder.extractWindowExpression(aliasExpr.child)
- wExpression.windowFunction match {
- case _: RowNumber | _: AggregateExpression | _: Rank | _: Lead | _: Lag |
- _: DenseRank =>
- allSupported = allSupported
- case _ =>
- logDebug(s"Not support window function: ${wExpression.getClass}")
- allSupported = false
- break
- }
- })
- }
- allSupported
- }
-
- override def supportStructType(): Boolean = true
-
- override def supportExpandExec(): Boolean = true
-
- override def excludeScanExecFromCollapsedStage(): Boolean =
- SQLConf.get
- .getConfString(GLUTEN_CLICKHOUSE_SEP_SCAN_RDD, GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT)
- .toBoolean
-
- /** Get the config prefix for each backend */
- override def getBackendConfigPrefix: String =
- GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME
-
- override def shuffleSupportedCodec(): Set[String] = GLUTEN_CLICKHOUSE_SHUFFLE_SUPPORTED_CODEC
- override def needOutputSchemaForPlan(): Boolean = true
-
- override def allowDecimalArithmetic: Boolean = !SQLConf.get.decimalOperationsAllowPrecisionLoss
-
- override def requiredInputFilePaths(): Boolean = true
-
- override def enableBloomFilterAggFallbackRule(): Boolean = false
-
- def maxShuffleReadRows(): Long = {
- SparkEnv.get.conf
- .getLong(GLUTEN_MAX_BLOCK_SIZE, GLUTEN_MAX_BLOCK_SIZE_DEFAULT)
- }
-
- def maxShuffleReadBytes(): Long = {
- SparkEnv.get.conf
- .getLong(GLUTEN_MAX_SHUFFLE_READ_BYTES, GLUTEN_MAX_SHUFFLE_READ_BYTES_DEFAULT)
- }
-
- override def supportWriteFilesExec(
- format: FileFormat,
- fields: Array[StructField],
- bucketSpec: Option[BucketSpec],
- options: Map[String, String]): ValidationResult =
- ValidationResult.notOk("CH backend is unsupported.")
-
- override def enableNativeWriteFiles(): Boolean = {
- GlutenConfig.getConf.enableNativeWriter.getOrElse(false)
- }
-
- override def mergeTwoPhasesHashBaseAggregateIfNeed(): Boolean = true
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBroadcastApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBroadcastApi.scala
deleted file mode 100644
index c5ab83ffe..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHBroadcastApi.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.backendsapi.BroadcastApi
-import io.glutenproject.execution.CHBroadcastBuildSideCache
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.rpc.GlutenDriverEndpoint
-
-class CHBroadcastApi extends BroadcastApi with Logging {
- override def cleanExecutionBroadcastTable(
- executionId: String,
- broadcastTableIds: java.util.Set[String]): Unit = {
- if (broadcastTableIds != null) {
- broadcastTableIds.forEach(
- resource_id => CHBroadcastBuildSideCache.invalidateBroadcastHashtable(resource_id))
- }
- }
-
- override def collectExecutionBroadcastTableId(executionId: String, buildTableId: String): Unit = {
- if (executionId != null) {
- GlutenDriverEndpoint.collectResources(executionId, buildTableId)
- } else {
- logWarning(
- s"Can't not trace broadcast hash table data $buildTableId" +
- s" because execution id is null." +
- s" Will clean up until expire time.")
- }
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHIteratorApi.scala
deleted file mode 100644
index 08786a00b..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHIteratorApi.scala
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.{GlutenConfig, GlutenNumaBindingInfo}
-import io.glutenproject.backendsapi.IteratorApi
-import io.glutenproject.execution._
-import io.glutenproject.expression.ConverterUtils
-import io.glutenproject.metrics.{GlutenTimeMetric, IMetrics, NativeMetrics}
-import io.glutenproject.substrait.plan.PlanNode
-import io.glutenproject.substrait.rel._
-import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat
-import io.glutenproject.utils.LogLevelUtil
-import io.glutenproject.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator, GeneralOutIterator}
-
-import org.apache.spark.{InterruptibleIterator, SparkConf, SparkContext, TaskContext}
-import org.apache.spark.affinity.CHAffinity
-import org.apache.spark.internal.Logging
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.connector.read.InputPartition
-import org.apache.spark.sql.execution.datasources.FilePartition
-import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.apache.spark.sql.utils.OASPackageBridge.InputMetricsWrapper
-import org.apache.spark.sql.vectorized.ColumnarBatch
-
-import java.lang.{Long => JLong}
-import java.net.URI
-import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Map => JMap}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil {
-
- private def getFileSchema(schema: StructType, names: Seq[String]): StructType = {
- val dataSchema = ArrayBuffer[StructField]()
- schema.foreach {
- field =>
- // case-insensitive schema matching
- val newField = names.find(_.equalsIgnoreCase(field.name)) match {
- case Some(name) => StructField(name, field.dataType, field.nullable, field.metadata)
- case _ => field
- }
- dataSchema += newField
- }
- StructType(dataSchema)
- }
-
- // only set file schema for text format table
- private def setFileSchemaForLocalFiles(
- localFilesNode: LocalFilesNode,
- scan: BasicScanExecTransformer): Unit = {
- if (scan.fileFormat == ReadFileFormat.TextReadFormat) {
- val names =
- ConverterUtils.collectAttributeNamesWithoutExprId(scan.outputAttributes())
- localFilesNode.setFileSchema(getFileSchema(scan.getDataSchema, names.asScala))
- }
- }
-
- override def genSplitInfo(
- partition: InputPartition,
- partitionSchema: StructType,
- fileFormat: ReadFileFormat,
- metadataColumnNames: Seq[String]): SplitInfo = {
- partition match {
- case p: GlutenMergeTreePartition =>
- val partLists = new JArrayList[String]()
- val starts = new JArrayList[JLong]()
- val lengths = new JArrayList[JLong]()
- p.partList
- .foreach(
- parts => {
- partLists.add(parts.name)
- starts.add(parts.start)
- lengths.add(parts.length)
- })
- ExtensionTableBuilder
- .makeExtensionTable(
- -1L,
- -1L,
- p.database,
- p.table,
- p.relativeTablePath,
- p.absoluteTablePath,
- p.orderByKey,
- p.lowCardKey,
- p.minmaxIndexKey,
- p.bfIndexKey,
- p.setIndexKey,
- p.primaryKey,
- partLists,
- starts,
- lengths,
- p.tableSchemaJson,
- p.clickhouseTableConfigs.asJava,
- CHAffinity.getNativeMergeTreePartitionLocations(p).toList.asJava
- )
- case f: FilePartition =>
- val paths = new JArrayList[String]()
- val starts = new JArrayList[JLong]()
- val lengths = new JArrayList[JLong]()
- val partitionColumns = new JArrayList[JMap[String, String]]
- f.files.foreach {
- file =>
- paths.add(new URI(file.filePath).toASCIIString)
- starts.add(JLong.valueOf(file.start))
- lengths.add(JLong.valueOf(file.length))
- // TODO: Support custom partition location
- val partitionColumn = new JHashMap[String, String]()
- partitionColumns.add(partitionColumn)
- }
- val preferredLocations =
- CHAffinity.getFilePartitionLocations(paths.asScala.toArray, f.preferredLocations())
- LocalFilesBuilder.makeLocalFiles(
- f.index,
- paths,
- starts,
- lengths,
- partitionColumns,
- new JArrayList[JMap[String, String]](),
- fileFormat,
- preferredLocations.toList.asJava)
- case _ =>
- throw new UnsupportedOperationException(s"Unsupported input partition: $partition.")
- }
- }
-
- /**
- * Generate native row partition.
- *
- * @return
- */
- override def genPartitions(
- wsCtx: WholeStageTransformContext,
- splitInfos: Seq[Seq[SplitInfo]],
- scans: Seq[BasicScanExecTransformer]): Seq[BaseGlutenPartition] = {
- // Only serialize plan once, save lots time when plan is complex.
- val planByteArray = wsCtx.root.toProtobuf.toByteArray
- splitInfos.zipWithIndex.map {
- case (splits, index) =>
- val files = ArrayBuffer[String]()
- val splitInfosByteArray = splits.zipWithIndex.map {
- case (split, i) =>
- split match {
- case filesNode: LocalFilesNode =>
- setFileSchemaForLocalFiles(filesNode, scans(i))
- filesNode.setFileReadProperties(mapAsJavaMap(scans(i).getProperties))
- filesNode.getPaths.forEach(f => files += f)
- filesNode.toProtobuf.toByteArray
- case extensionTableNode: ExtensionTableNode =>
- extensionTableNode.getPartList.forEach(
- name => files += extensionTableNode.getAbsolutePath + "/" + name)
- extensionTableNode.toProtobuf.toByteArray
- }
- }
-
- GlutenPartition(
- index,
- planByteArray,
- splitInfosByteArray.toArray,
- locations = splits.flatMap(_.preferredLocations().asScala).toArray,
- files.toArray
- )
- }
- }
-
- /**
- * Generate Iterator[ColumnarBatch] for first stage.
- *
- * @return
- */
- override def genFirstStageIterator(
- inputPartition: BaseGlutenPartition,
- context: TaskContext,
- pipelineTime: SQLMetric,
- updateInputMetrics: InputMetricsWrapper => Unit,
- updateNativeMetrics: IMetrics => Unit,
- partitionIndex: Int,
- inputIterators: Seq[Iterator[ColumnarBatch]] = Seq()
- ): Iterator[ColumnarBatch] = {
-
- assert(
- inputPartition.isInstanceOf[GlutenPartition],
- "CH backend only accepts GlutenPartition in GlutenWholeStageColumnarRDD.")
-
- val transKernel = new CHNativeExpressionEvaluator()
- val inBatchIters = new JArrayList[GeneralInIterator](inputIterators.map {
- iter => new ColumnarNativeIterator(CHIteratorApi.genCloseableColumnBatchIterator(iter).asJava)
- }.asJava)
-
- val splitInfoByteArray = inputPartition
- .asInstanceOf[GlutenPartition]
- .splitInfosByteArray
- val resIter: GeneralOutIterator =
- transKernel.createKernelWithBatchIterator(
- inputPartition.plan,
- splitInfoByteArray,
- inBatchIters,
- false)
-
- context.addTaskCompletionListener[Unit](_ => resIter.close())
- val iter = new Iterator[Any] {
- private val inputMetrics = context.taskMetrics().inputMetrics
- private var outputRowCount = 0L
- private var outputVectorCount = 0L
- private var metricsUpdated = false
-
- override def hasNext: Boolean = {
- val res = resIter.hasNext
- // avoid to collect native metrics more than once, 'hasNext' is a idempotent operation
- if (!res && !metricsUpdated) {
- val nativeMetrics = resIter.getMetrics.asInstanceOf[NativeMetrics]
- nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount)
- updateNativeMetrics(nativeMetrics)
- updateInputMetrics(inputMetrics)
- metricsUpdated = true
- }
- res
- }
-
- override def next(): Any = {
- val cb = resIter.next()
- outputVectorCount += 1
- outputRowCount += cb.numRows()
- cb
- }
- }
-
- // TODO: SPARK-25083 remove the type erasure hack in data source scan
- new InterruptibleIterator(
- context,
- new CloseableCHColumnBatchIterator(
- iter.asInstanceOf[Iterator[ColumnarBatch]],
- Some(pipelineTime)))
- }
-
- // Generate Iterator[ColumnarBatch] for final stage.
- // scalastyle:off argcount
- override def genFinalStageIterator(
- context: TaskContext,
- inputIterators: Seq[Iterator[ColumnarBatch]],
- numaBindingInfo: GlutenNumaBindingInfo,
- sparkConf: SparkConf,
- rootNode: PlanNode,
- pipelineTime: SQLMetric,
- updateNativeMetrics: IMetrics => Unit,
- partitionIndex: Int,
- materializeInput: Boolean): Iterator[ColumnarBatch] = {
- // scalastyle:on argcount
- GlutenConfig.getConf
-
- val transKernel = new CHNativeExpressionEvaluator()
- val columnarNativeIterator =
- new JArrayList[GeneralInIterator](inputIterators.map {
- iter =>
- new ColumnarNativeIterator(CHIteratorApi.genCloseableColumnBatchIterator(iter).asJava)
- }.asJava)
- // we need to complete dependency RDD's firstly
- val nativeIterator = transKernel.createKernelWithBatchIterator(
- rootNode.toProtobuf.toByteArray,
- // Final iterator does not contain scan split, so pass empty split info to native here.
- new Array[Array[Byte]](0),
- columnarNativeIterator,
- materializeInput
- )
-
- val resIter = new Iterator[ColumnarBatch] {
- private var outputRowCount = 0L
- private var outputVectorCount = 0L
- private var metricsUpdated = false
-
- override def hasNext: Boolean = {
- val res = nativeIterator.hasNext
- // avoid to collect native metrics more than once, 'hasNext' is a idempotent operation
- if (!res && !metricsUpdated) {
- val nativeMetrics = nativeIterator.getMetrics.asInstanceOf[NativeMetrics]
- nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount)
- updateNativeMetrics(nativeMetrics)
- metricsUpdated = true
- }
- res
- }
-
- override def next(): ColumnarBatch = {
- val cb = nativeIterator.next()
- outputVectorCount += 1
- outputRowCount += cb.numRows()
- cb
- }
- }
- var closed = false
-
- def close(): Unit = {
- closed = true
- nativeIterator.close()
- // relationHolder.clear()
- }
-
- context.addTaskCompletionListener[Unit](_ => close())
- new CloseableCHColumnBatchIterator(resIter, Some(pipelineTime))
- }
-
- /** Generate Native FileScanRDD, currently only for ClickHouse Backend. */
- override def genNativeFileScanRDD(
- sparkContext: SparkContext,
- wsCtx: WholeStageTransformContext,
- splitInfos: Seq[SplitInfo],
- scan: BasicScanExecTransformer,
- numOutputRows: SQLMetric,
- numOutputBatches: SQLMetric,
- scanTime: SQLMetric): RDD[ColumnarBatch] = {
- val substraitPlanPartition = GlutenTimeMetric.withMillisTime {
- val planByteArray = wsCtx.root.toProtobuf.toByteArray
- splitInfos.zipWithIndex.map {
- case (splitInfo, index) =>
- val splitInfoByteArray = splitInfo match {
- case filesNode: LocalFilesNode =>
- setFileSchemaForLocalFiles(filesNode, scan)
- filesNode.setFileReadProperties(mapAsJavaMap(scan.getProperties))
- filesNode.toProtobuf.toByteArray
- case extensionTableNode: ExtensionTableNode =>
- extensionTableNode.toProtobuf.toByteArray
- }
-
- GlutenPartition(
- index,
- planByteArray,
- Array(splitInfoByteArray),
- locations = splitInfo.preferredLocations().asScala.toArray)
- }
- }(t => logInfo(s"Generating the Substrait plan took: $t ms."))
-
- new NativeFileScanColumnarRDD(
- sparkContext,
- substraitPlanPartition,
- numOutputRows,
- numOutputBatches,
- scanTime)
- }
-}
-
-object CHIteratorApi {
-
- /** Generate closeable ColumnBatch iterator. */
- def genCloseableColumnBatchIterator(iter: Iterator[ColumnarBatch]): Iterator[ColumnarBatch] = {
- iter match {
- case _: CloseableCHColumnBatchIterator => iter
- case _ => new CloseableCHColumnBatchIterator(iter)
- }
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHListenerApi.scala
deleted file mode 100644
index 5a1b529d5..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHListenerApi.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.GlutenConfig
-import io.glutenproject.backendsapi.ListenerApi
-import io.glutenproject.execution.CHBroadcastBuildSideCache
-import io.glutenproject.execution.datasource.{GlutenOrcWriterInjects, GlutenParquetWriterInjects, GlutenRowSplitter}
-import io.glutenproject.expression.UDFMappings
-import io.glutenproject.vectorized.{CHNativeExpressionEvaluator, JniLibLoader}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.execution.datasources.v1._
-
-import org.apache.commons.lang3.StringUtils
-
-import java.util.TimeZone
-
-class CHListenerApi extends ListenerApi with Logging {
-
- override def onDriverStart(conf: SparkConf): Unit = initialize(conf, isDriver = true)
-
- override def onDriverShutdown(): Unit = shutdown()
-
- override def onExecutorStart(conf: SparkConf): Unit = initialize(conf, isDriver = false)
-
- override def onExecutorShutdown(): Unit = shutdown()
-
- private def initialize(conf: SparkConf, isDriver: Boolean): Unit = {
- val libPath = conf.get(GlutenConfig.GLUTEN_LIB_PATH, StringUtils.EMPTY)
- if (StringUtils.isBlank(libPath)) {
- throw new IllegalArgumentException(
- "Please set spark.gluten.sql.columnar.libpath to enable clickhouse backend")
- }
- if (isDriver) {
- JniLibLoader.loadFromPath(libPath, true)
- } else {
- val executorLibPath = conf.get(GlutenConfig.GLUTEN_EXECUTOR_LIB_PATH, libPath)
- JniLibLoader.loadFromPath(executorLibPath, true)
- }
- // Add configs
- conf.set(
- s"${CHBackendSettings.getBackendConfigPrefix}.runtime_config.timezone",
- conf.get("spark.sql.session.timeZone", TimeZone.getDefault.getID))
- conf.set(
- s"${CHBackendSettings.getBackendConfigPrefix}.runtime_config" +
- s".local_engine.settings.log_processors_profiles",
- "true")
-
- // Load supported hive/python/scala udfs
- UDFMappings.loadFromSparkConf(conf)
-
- val initKernel = new CHNativeExpressionEvaluator()
- initKernel.initNative(conf)
-
- // inject backend-specific implementations to override spark classes
- // FIXME: The following set instances twice in local mode?
- GlutenParquetWriterInjects.setInstance(new CHParquetWriterInjects())
- GlutenOrcWriterInjects.setInstance(new CHOrcWriterInjects())
- GlutenMergeTreeWriterInjects.setInstance(new CHMergeTreeWriterInjects())
- GlutenRowSplitter.setInstance(new CHRowSplitter())
- }
-
- private def shutdown(): Unit = {
- CHBroadcastBuildSideCache.cleanAll()
- val kernel = new CHNativeExpressionEvaluator()
- kernel.finalizeNative()
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHMetricsApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHMetricsApi.scala
deleted file mode 100644
index 0157b370f..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHMetricsApi.scala
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.backendsapi.MetricsApi
-import io.glutenproject.metrics._
-import io.glutenproject.substrait.{AggregationParams, JoinParams}
-import io.glutenproject.utils.LogLevelUtil
-
-import org.apache.spark.SparkContext
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
-
-import java.lang.{Long => JLong}
-import java.util.{List => JList, Map => JMap}
-
-class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil {
- override def metricsUpdatingFunction(
- child: SparkPlan,
- relMap: JMap[JLong, JList[JLong]],
- joinParamsMap: JMap[JLong, JoinParams],
- aggParamsMap: JMap[JLong, AggregationParams]): IMetrics => Unit = {
- MetricsUtil.updateNativeMetrics(child, relMap, joinParamsMap, aggParamsMap)
- }
-
- override def genInputIteratorTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] = {
- Map(
- "iterReadTime" -> SQLMetrics.createTimingMetric(
- sparkContext,
- "time of reading from iterator"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "fillingRightJoinSideTime" -> SQLMetrics.createTimingMetric(
- sparkContext,
- "filling right join side time")
- )
- }
-
- override def genInputIteratorTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = {
- InputIteratorMetricsUpdater(metrics)
- }
-
- override def genBatchScanTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputVectors" -> SQLMetrics.createMetric(sparkContext, "number of input vectors"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "rawInputRows" -> SQLMetrics.createMetric(sparkContext, "number of raw input rows"),
- "rawInputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of raw input bytes"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "scanTime" -> SQLMetrics.createTimingMetric(sparkContext, "scan time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time")
- )
-
- override def genBatchScanTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new BatchScanMetricsUpdater(metrics)
-
- override def genHiveTableScanTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputVectors" -> SQLMetrics.createMetric(sparkContext, "number of input vectors"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "rawInputRows" -> SQLMetrics.createMetric(sparkContext, "number of raw input rows"),
- "rawInputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of raw input bytes"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "scanTime" -> SQLMetrics.createTimingMetric(sparkContext, "scan time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
- "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
- "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
- "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
- "pruningTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "readBytes" -> SQLMetrics.createMetric(sparkContext, "number of read bytes")
- )
-
- override def genHiveTableScanTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new HiveTableScanMetricsUpdater(metrics)
-
- override def genFileSourceScanTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputVectors" -> SQLMetrics.createMetric(sparkContext, "number of input vectors"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "rawInputRows" -> SQLMetrics.createMetric(sparkContext, "number of raw input rows"),
- "rawInputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of raw input bytes"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "scanTime" -> SQLMetrics.createTimingMetric(sparkContext, "scan time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
- "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
- "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
- "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
- "pruningTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "selectedMarksPk" -> SQLMetrics.createMetric(sparkContext, "selected marks primary"),
- "selectedMarks" -> SQLMetrics.createMetric(sparkContext, "selected marks"),
- "totalMarksPk" -> SQLMetrics.createMetric(sparkContext, "total marks primary")
- )
-
- override def genFileSourceScanTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new FileSourceScanMetricsUpdater(metrics)
-
- override def genFilterTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genFilterTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater =
- new FilterMetricsUpdater(metrics)
-
- override def genProjectTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genProjectTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new ProjectMetricsUpdater(metrics)
-
- override def genHashAggregateTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "resizeInputRows" -> SQLMetrics.createMetric(sparkContext, "number of resize input rows"),
- "resizeOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of resize output rows"),
- "aggregatingTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "time of aggregating"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genHashAggregateTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater =
- new HashAggregateMetricsUpdater(metrics)
-
- override def genExpandTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genExpandTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater =
- new ExpandMetricsUpdater(metrics)
-
- override def genCustomExpandMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
-
- override def genColumnarShuffleExchangeMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
- "bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle bytes spilled"),
- "computePidTime" -> SQLMetrics.createNanoTimingMetric(
- sparkContext,
- "totaltime to compute pid"),
- "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to split"),
- "IOTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to disk io"),
- "serializeTime" -> SQLMetrics.createNanoTimingMetric(
- sparkContext,
- "totaltime to block serialization"),
- "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to spill"),
- "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to compress"),
- "prepareTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to prepare"),
- "avgReadBatchNumRows" -> SQLMetrics
- .createAverageMetric(sparkContext, "avg read batch num rows"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "numOutputRows" -> SQLMetrics
- .createMetric(sparkContext, "number of output rows"),
- "inputBatches" -> SQLMetrics
- .createMetric(sparkContext, "number of input batches"),
- "outputBatches" -> SQLMetrics
- .createMetric(sparkContext, "number of output batches")
- )
-
- override def genWindowTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genWindowTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater =
- new WindowMetricsUpdater(metrics)
-
- override def genColumnarToRowMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "numInputBatches" -> SQLMetrics.createMetric(sparkContext, "number of input batches"),
- "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to convert")
- )
-
- override def genRowToColumnarMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "numOutputBatches" -> SQLMetrics.createMetric(sparkContext, "number of output batches"),
- "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time to convert")
- )
-
- override def genLimitTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genLimitTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater =
- new LimitMetricsUpdater(metrics)
-
- override def genSortTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time")
- )
-
- override def genSortTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater =
- new SortMetricsUpdater(metrics)
-
- override def genSortMergeJoinTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map.empty
-
- override def genSortMergeJoinTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new SortMergeJoinMetricsUpdater(metrics)
-
- override def genColumnarBroadcastExchangeMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "collectTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to collect"),
- "broadcastTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to broadcast")
- )
-
- override def genColumnarSubqueryBroadcastMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "dataSize" -> SQLMetrics.createMetric(sparkContext, "data size (bytes)"),
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "collectTime" -> SQLMetrics.createMetric(sparkContext, "time to collect (ms)")
- )
-
- override def genHashJoinTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] =
- Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"),
- "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"),
- "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"),
- "inputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of input bytes"),
- "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"),
- "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"),
- "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"),
- "streamPreProjectionTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "time of stream side preProjection"),
- "buildPreProjectionTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "time of build side preProjection"),
- "postProjectTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "time of postProjection"),
- "probeTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "time of probe"),
- "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time"),
- "fillingRightJoinSideTime" -> SQLMetrics.createTimingMetric(
- sparkContext,
- "filling right join side time"),
- "conditionTime" -> SQLMetrics.createTimingMetric(sparkContext, "join condition time")
- )
-
- override def genHashJoinTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = new HashJoinMetricsUpdater(metrics)
-
- override def genNestedLoopJoinTransformerMetrics(
- sparkContext: SparkContext): Map[String, SQLMetric] = {
- throw new UnsupportedOperationException(
- s"NestedLoopJoinTransformer metrics update is not supported in CH backend")
- }
-
- override def genNestedLoopJoinTransformerMetricsUpdater(
- metrics: Map[String, SQLMetric]): MetricsUpdater = {
- throw new UnsupportedOperationException(
- s"NestedLoopJoinTransformer metrics update is not supported in CH backend")
- }
-
- def genWriteFilesTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = {
- throw new UnsupportedOperationException(
- s"WriteFilesTransformer metrics update is not supported in CH backend")
- }
-
- def genWriteFilesTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = {
- throw new UnsupportedOperationException(
- s"WriteFilesTransformer metrics update is not supported in CH backend")
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
deleted file mode 100644
index eef27665f..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
+++ /dev/null
@@ -1,816 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.GlutenConfig
-import io.glutenproject.backendsapi.{BackendsApiManager, SparkPlanExecApi}
-import io.glutenproject.exception.GlutenNotSupportException
-import io.glutenproject.execution._
-import io.glutenproject.expression._
-import io.glutenproject.expression.ConverterUtils.FunctionConfig
-import io.glutenproject.extension.{CountDistinctWithoutExpand, FallbackBroadcastHashJoin, FallbackBroadcastHashJoinPrepQueryStage}
-import io.glutenproject.extension.columnar.AddTransformHintRule
-import io.glutenproject.extension.columnar.MiscColumnarRules.TransformPreOverrides
-import io.glutenproject.substrait.expression.{ExpressionBuilder, ExpressionNode, WindowFunctionNode}
-import io.glutenproject.utils.CHJoinValidateUtil
-import io.glutenproject.vectorized.CHColumnarBatchSerializer
-
-import org.apache.spark.{ShuffleDependency, SparkException}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.serializer.Serializer
-import org.apache.spark.shuffle.{GenShuffleWriterParameters, GlutenShuffleWriterWrapper, HashPartitioningWrapper}
-import org.apache.spark.shuffle.utils.CHShuffleUtil
-import org.apache.spark.sql.{SparkSession, Strategy}
-import org.apache.spark.sql.catalyst.CHAggregateFunctionRewriteRule
-import org.apache.spark.sql.catalyst.catalog.BucketSpec
-import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.optimizer.BuildSide
-import org.apache.spark.sql.catalyst.plans.JoinType
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, HashPartitioning, Partitioning, RangePartitioning}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.delta.files.TahoeFileIndex
-import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec
-import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, WriteFilesExec}
-import org.apache.spark.sql.execution.datasources.GlutenWriterColumnarRules.NativeWritePostRule
-import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
-import org.apache.spark.sql.execution.joins.{BuildSideRelation, ClickHouseBuildSideRelation, HashedRelationBroadcastMode}
-import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.execution.utils.{CHExecUtil, PushDownUtil}
-import org.apache.spark.sql.extension.{CommonSubexpressionEliminateRule, RewriteDateTimestampComparisonRule}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.vectorized.ColumnarBatch
-
-import com.google.common.collect.Lists
-import org.apache.commons.lang3.ClassUtils
-
-import java.lang.{Long => JLong}
-import java.util.{ArrayList => JArrayList, List => JList, Map => JMap}
-
-import scala.collection.mutable.ArrayBuffer
-
-class CHSparkPlanExecApi extends SparkPlanExecApi {
-
- /** Transform GetArrayItem to Substrait. */
- override def genGetArrayItemExpressionNode(
- substraitExprName: String,
- functionMap: JMap[String, JLong],
- leftNode: ExpressionNode,
- rightNode: ExpressionNode,
- original: GetArrayItem): ExpressionNode = {
- val functionName = ConverterUtils.makeFuncName(
- substraitExprName,
- Seq(original.left.dataType, original.right.dataType),
- FunctionConfig.OPT)
- val exprNodes = Lists.newArrayList(leftNode, rightNode)
- ExpressionBuilder.makeScalarFunction(
- ExpressionBuilder.newScalarFunction(functionMap, functionName),
- exprNodes,
- ConverterUtils.getTypeNode(original.dataType, original.nullable))
- }
-
- /**
- * Generate ColumnarToRowExecBase.
- *
- * @param child
- * @return
- */
- override def genColumnarToRowExec(child: SparkPlan): ColumnarToRowExecBase = {
- CHColumnarToRowExec(child)
- }
-
- /**
- * Generate RowToColumnarExec.
- *
- * @param child
- * @return
- */
- override def genRowToColumnarExec(child: SparkPlan): RowToColumnarExecBase = {
- RowToCHNativeColumnarExec(child)
- }
-
- override def genProjectExecTransformer(
- projectList: Seq[NamedExpression],
- child: SparkPlan): ProjectExecTransformer = {
- def processProjectExecTransformer(projectList: Seq[NamedExpression]): Seq[NamedExpression] = {
- // When there is a MergeScalarSubqueries which will create the named_struct with the
- // same name, looks like {'bloomFilter', BF1, 'bloomFilter', BF2}
- // or {'count(1)', count(1)#111L, 'avg(a)', avg(a)#222L, 'count(1)', count(1)#333L},
- // it will cause problem for ClickHouse backend,
- // which cannot tolerate duplicate type names in struct type,
- // so we need to rename 'nameExpr' in the named_struct to make them unique
- // after executing the MergeScalarSubqueries.
- var needToReplace = false
- val newProjectList = projectList.map {
- case alias @ Alias(cns @ CreateNamedStruct(children: Seq[Expression]), "mergedValue") =>
- // check whether there are some duplicate names
- if (cns.nameExprs.distinct.size == cns.nameExprs.size) {
- alias
- } else {
- val newChildren = children
- .grouped(2)
- .flatMap {
- case Seq(name: Literal, value: NamedExpression) =>
- val newLiteral = Literal(name.toString() + "#" + value.exprId.id)
- Seq(newLiteral, value)
- case Seq(name, value) => Seq(name, value)
- }
- .toSeq
- needToReplace = true
- Alias.apply(CreateNamedStruct(newChildren), "mergedValue")(alias.exprId)
- }
- case other: NamedExpression => other
- }
-
- if (!needToReplace) {
- projectList
- } else {
- newProjectList
- }
- }
-
- ProjectExecTransformer.createUnsafe(processProjectExecTransformer(projectList), child)
- }
-
- /**
- * Generate FilterExecTransformer.
- *
- * @param condition
- * : the filter condition
- * @param child
- * : the child of FilterExec
- * @return
- * the transformer of FilterExec
- */
- override def genFilterExecTransformer(
- condition: Expression,
- child: SparkPlan): FilterExecTransformerBase = {
-
- def checkMergeTreeFileFormat(relation: HadoopFsRelation): Boolean = {
- relation.location.isInstanceOf[TahoeFileIndex] &&
- relation.fileFormat.isInstanceOf[DeltaMergeTreeFileFormat]
- }
-
- child match {
- case scan: FileSourceScanExec if (checkMergeTreeFileFormat(scan.relation)) =>
- // For the validation phase of the AddTransformHintRule
- CHFilterExecTransformer(condition, child)
- case scan: FileSourceScanExecTransformerBase if (checkMergeTreeFileFormat(scan.relation)) =>
- // For the transform phase, the FileSourceScanExec is already transformed
- CHFilterExecTransformer(condition, child)
- case _ =>
- FilterExecTransformer(condition, child)
- }
- }
-
- /** Generate HashAggregateExecTransformer. */
- override def genHashAggregateExecTransformer(
- requiredChildDistributionExpressions: Option[Seq[Expression]],
- groupingExpressions: Seq[NamedExpression],
- aggregateExpressions: Seq[AggregateExpression],
- aggregateAttributes: Seq[Attribute],
- initialInputBufferOffset: Int,
- resultExpressions: Seq[NamedExpression],
- child: SparkPlan): HashAggregateExecBaseTransformer =
- CHHashAggregateExecTransformer(
- requiredChildDistributionExpressions,
- groupingExpressions,
- aggregateExpressions,
- aggregateAttributes,
- initialInputBufferOffset,
- resultExpressions,
- child)
-
- /** Generate HashAggregateExecPullOutHelper */
- override def genHashAggregateExecPullOutHelper(
- groupingExpressions: Seq[NamedExpression],
- aggregateExpressions: Seq[AggregateExpression],
- aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper =
- CHHashAggregateExecPullOutHelper(groupingExpressions, aggregateExpressions, aggregateAttributes)
-
- /**
- * If there are expressions (not field reference) in the partitioning's children, add a projection
- * before shuffle exchange and make a new partitioning with the old expressions replaced by the
- * result columns from the projection.
- */
- private def addProjectionForShuffleExchange(
- plan: ShuffleExchangeExec): (Int, Partitioning, SparkPlan) = {
- def selectExpressions(
- exprs: Seq[Expression],
- attributes: Seq[Attribute]): (Seq[NamedExpression], Seq[Int]) = {
- var expressionPos = Seq[Int]()
- var projectExpressions = Seq[NamedExpression]()
-
- exprs.foreach(
- expr => {
- if (!expr.isInstanceOf[AttributeReference]) {
- val n = projectExpressions.size
- val namedExpression = Alias(expr, s"projected_partitioning_value_$n")()
- projectExpressions = projectExpressions :+ namedExpression
- expressionPos = expressionPos :+ (attributes.size + n)
- } else {
- // the new projected columns are appended at the end
- expressionPos = expressionPos :+ BindReferences
- .bindReference(expr, attributes)
- .asInstanceOf[BoundReference]
- .ordinal
- }
- })
- (projectExpressions, expressionPos)
- }
-
- plan.outputPartitioning match {
- case HashPartitioning(exprs, numPartitions) =>
- val (projectExpressions, newExpressionsPosition) = {
- selectExpressions(
- exprs,
- BackendsApiManager.getTransformerApiInstance
- .getPlanOutput(plan.child))
- }
- if (projectExpressions.isEmpty) {
- return (0, plan.outputPartitioning, plan.child)
- }
- // FIXME: The operation happens inside ReplaceSingleNode().
- // Caller may not know it adds project on top of the shuffle.
- val project = TransformPreOverrides().apply(
- AddTransformHintRule().apply(
- ProjectExec(plan.child.output ++ projectExpressions, plan.child)))
- var newExprs = Seq[Expression]()
- for (i <- exprs.indices) {
- val pos = newExpressionsPosition(i)
- newExprs = newExprs :+ project.output(pos)
- }
- (
- projectExpressions.size,
- new HashPartitioningWrapper(exprs, newExprs, numPartitions),
- project)
- case RangePartitioning(orderings, numPartitions) =>
- val exprs = orderings.map(ordering => ordering.child)
- val (projectExpressions, newExpressionsPosition) = {
- selectExpressions(
- exprs,
- BackendsApiManager.getTransformerApiInstance
- .getPlanOutput(plan.child))
- }
- if (projectExpressions.isEmpty) {
- return (0, plan.outputPartitioning, plan.child)
- }
- // FIXME: The operation happens inside ReplaceSingleNode().
- // Caller may not know it adds project on top of the shuffle.
- val project = TransformPreOverrides().apply(
- AddTransformHintRule().apply(
- ProjectExec(plan.child.output ++ projectExpressions, plan.child)))
- var newOrderings = Seq[SortOrder]()
- for (i <- orderings.indices) {
- val oldOrdering = orderings(i)
- val pos = newExpressionsPosition(i)
- val ordering = SortOrder(
- project.output(pos),
- oldOrdering.direction,
- oldOrdering.nullOrdering,
- oldOrdering.sameOrderExpressions)
- newOrderings = newOrderings :+ ordering
- }
- (projectExpressions.size, RangePartitioning(newOrderings, numPartitions), project)
- case _ =>
- // no change for other cases
- (0, plan.outputPartitioning, plan.child)
- }
- }
-
- override def genColumnarShuffleExchange(
- shuffle: ShuffleExchangeExec,
- child: SparkPlan): SparkPlan = {
- if (
- BackendsApiManager.getSettings.supportShuffleWithProject(
- shuffle.outputPartitioning,
- shuffle.child)
- ) {
- val (projectColumnNumber, newPartitioning, newChild) =
- addProjectionForShuffleExchange(shuffle)
-
- if (projectColumnNumber != 0) {
- if (newChild.supportsColumnar) {
- val newPlan = ShuffleExchangeExec(newPartitioning, newChild, shuffle.shuffleOrigin)
- // the new projections columns are appended at the end.
- ColumnarShuffleExchangeExec(
- newPlan,
- newChild,
- newChild.output.dropRight(projectColumnNumber))
- } else {
- // It's the case that partitioning expressions could be offloaded into native.
- shuffle.withNewChildren(Seq(child))
- }
- } else {
- ColumnarShuffleExchangeExec(shuffle, child, null)
- }
- } else {
- ColumnarShuffleExchangeExec(shuffle, child, null)
- }
- }
-
- /** Generate ShuffledHashJoinExecTransformer. */
- def genShuffledHashJoinExecTransformer(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
- joinType: JoinType,
- buildSide: BuildSide,
- condition: Option[Expression],
- left: SparkPlan,
- right: SparkPlan,
- isSkewJoin: Boolean): ShuffledHashJoinExecTransformerBase =
- CHShuffledHashJoinExecTransformer(
- leftKeys,
- rightKeys,
- joinType,
- buildSide,
- condition,
- left,
- right,
- isSkewJoin)
-
- /** Generate BroadcastHashJoinExecTransformer. */
- def genBroadcastHashJoinExecTransformer(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
- joinType: JoinType,
- buildSide: BuildSide,
- condition: Option[Expression],
- left: SparkPlan,
- right: SparkPlan,
- isNullAwareAntiJoin: Boolean = false): BroadcastHashJoinExecTransformerBase =
- CHBroadcastHashJoinExecTransformer(
- leftKeys,
- rightKeys,
- joinType,
- buildSide,
- condition,
- left,
- right,
- isNullAwareAntiJoin)
-
- /** Generate CartesianProductExecTransformer */
- override def genCartesianProductExecTransformer(
- left: SparkPlan,
- right: SparkPlan,
- condition: Option[Expression]): CartesianProductExecTransformer =
- throw new GlutenNotSupportException(
- "CartesianProductExecTransformer is not supported in ch backend.")
-
- override def genBroadcastNestedLoopJoinExecTransformer(
- left: SparkPlan,
- right: SparkPlan,
- buildSide: BuildSide,
- joinType: JoinType,
- condition: Option[Expression]): BroadcastNestedLoopJoinExecTransformer =
- throw new GlutenNotSupportException(
- "BroadcastNestedLoopJoinExecTransformer is not supported in ch backend.")
-
- /** Generate an expression transformer to transform GetMapValue to Substrait. */
- def genGetMapValueTransformer(
- substraitExprName: String,
- left: ExpressionTransformer,
- right: ExpressionTransformer,
- original: GetMapValue): ExpressionTransformer =
- GetMapValueTransformer(substraitExprName, left, right, original.failOnError, original)
-
- override def genRandTransformer(
- substraitExprName: String,
- explicitSeed: ExpressionTransformer,
- original: Rand): ExpressionTransformer = {
- GenericExpressionTransformer(substraitExprName, Seq(explicitSeed), original)
- }
-
- /**
- * Generate ShuffleDependency for ColumnarShuffleExchangeExec.
- *
- * @return
- */
- // scalastyle:off argcount
- override def genShuffleDependency(
- rdd: RDD[ColumnarBatch],
- childOutputAttributes: Seq[Attribute],
- projectOutputAttributes: Seq[Attribute],
- newPartitioning: Partitioning,
- serializer: Serializer,
- writeMetrics: Map[String, SQLMetric],
- metrics: Map[String, SQLMetric]
- ): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = {
- CHExecUtil.genShuffleDependency(
- rdd,
- childOutputAttributes,
- projectOutputAttributes,
- newPartitioning,
- serializer,
- writeMetrics,
- metrics
- )
- }
- // scalastyle:on argcount
-
- /**
- * Generate ColumnarShuffleWriter for ColumnarShuffleManager.
- *
- * @return
- */
- override def genColumnarShuffleWriter[K, V](
- parameters: GenShuffleWriterParameters[K, V]): GlutenShuffleWriterWrapper[K, V] = {
- CHShuffleUtil.genColumnarShuffleWriter(parameters)
- }
-
- /**
- * Generate ColumnarBatchSerializer for ColumnarShuffleExchangeExec.
- *
- * @return
- */
- override def createColumnarBatchSerializer(
- schema: StructType,
- metrics: Map[String, SQLMetric]): Serializer = {
- val readBatchNumRows = metrics("avgReadBatchNumRows")
- val numOutputRows = metrics("numOutputRows")
- val dataSize = metrics("dataSize")
- if (GlutenConfig.getConf.isUseCelebornShuffleManager) {
- val clazz = ClassUtils.getClass("org.apache.spark.shuffle.CHCelebornColumnarBatchSerializer")
- val constructor =
- clazz.getConstructor(classOf[SQLMetric], classOf[SQLMetric], classOf[SQLMetric])
- constructor.newInstance(readBatchNumRows, numOutputRows, dataSize).asInstanceOf[Serializer]
- } else {
- new CHColumnarBatchSerializer(readBatchNumRows, numOutputRows, dataSize)
- }
- }
-
- /** Create broadcast relation for BroadcastExchangeExec */
- override def createBroadcastRelation(
- mode: BroadcastMode,
- child: SparkPlan,
- numOutputRows: SQLMetric,
- dataSize: SQLMetric): BuildSideRelation = {
- val hashedRelationBroadcastMode = mode.asInstanceOf[HashedRelationBroadcastMode]
- val (newChild, newOutput, newBuildKeys) =
- if (
- hashedRelationBroadcastMode.key
- .forall(k => k.isInstanceOf[AttributeReference] || k.isInstanceOf[BoundReference])
- ) {
- (child, child.output, Seq.empty[Expression])
- } else {
- // pre projection in case of expression join keys
- val buildKeys = hashedRelationBroadcastMode.key
- val appendedProjections = new ArrayBuffer[NamedExpression]()
- val preProjectionBuildKeys = buildKeys.zipWithIndex.map {
- case (e, idx) =>
- e match {
- case b: BoundReference => child.output(b.ordinal)
- case o: Expression =>
- val newExpr = Alias(o, "col_" + idx)()
- appendedProjections += newExpr
- newExpr
- }
- }
-
- def wrapChild(child: SparkPlan): WholeStageTransformer = {
- val childWithAdapter = ColumnarCollapseTransformStages.wrapInputIteratorTransformer(child)
- WholeStageTransformer(
- ProjectExecTransformer(child.output ++ appendedProjections, childWithAdapter))(
- ColumnarCollapseTransformStages.transformStageCounter.incrementAndGet()
- )
- }
-
- val newChild = child match {
- case wt: WholeStageTransformer =>
- wt.withNewChildren(
- Seq(ProjectExecTransformer(child.output ++ appendedProjections, wt.child)))
- case w: WholeStageCodegenExec =>
- w.withNewChildren(Seq(ProjectExec(child.output ++ appendedProjections, w.child)))
- case r: AQEShuffleReadExec if r.supportsColumnar =>
- // when aqe is open
- // TODO: remove this after pushdowning preprojection
- wrapChild(r)
- case r2c: RowToCHNativeColumnarExec =>
- wrapChild(r2c)
- case union: ColumnarUnionExec =>
- wrapChild(union)
- case other =>
- throw new GlutenNotSupportException(
- s"Not supported operator ${other.nodeName} for BroadcastRelation")
- }
- (newChild, (child.output ++ appendedProjections).map(_.toAttribute), preProjectionBuildKeys)
- }
- val countsAndBytes =
- CHExecUtil.buildSideRDD(dataSize, newChild).collect
-
- val batches = countsAndBytes.map(_._2)
- val rawSize = dataSize.value
- if (rawSize >= BroadcastExchangeExec.MAX_BROADCAST_TABLE_BYTES) {
- throw new SparkException(
- s"Cannot broadcast the table that is larger than 8GB: ${rawSize >> 30} GB")
- }
- val rowCount = countsAndBytes.map(_._1).sum
- numOutputRows += rowCount
- ClickHouseBuildSideRelation(mode, newOutput, batches.flatten, rowCount, newBuildKeys)
- }
-
- /**
- * Generate extended DataSourceV2 Strategies. Currently only for ClickHouse backend.
- *
- * @return
- */
- override def genExtendedDataSourceV2Strategies(): List[SparkSession => Strategy] = {
- List.empty
- }
-
- /**
- * Generate extended query stage preparation rules.
- *
- * @return
- */
- override def genExtendedQueryStagePrepRules(): List[SparkSession => Rule[SparkPlan]] =
- List(spark => FallbackBroadcastHashJoinPrepQueryStage(spark))
-
- /**
- * Generate extended Analyzers. Currently only for ClickHouse backend.
- *
- * @return
- */
- override def genExtendedAnalyzers(): List[SparkSession => Rule[LogicalPlan]] = {
- List(spark => new RewriteDateTimestampComparisonRule(spark, spark.sessionState.conf))
- }
-
- /**
- * Generate extended Optimizers.
- *
- * @return
- */
- override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] = {
- List(
- spark => new CommonSubexpressionEliminateRule(spark, spark.sessionState.conf),
- spark => CHAggregateFunctionRewriteRule(spark),
- _ => CountDistinctWithoutExpand
- )
- }
-
- /**
- * Generate extended columnar pre-rules, in the validation phase.
- *
- * @return
- */
- override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] =
- List(spark => FallbackBroadcastHashJoin(spark))
-
- /**
- * Generate extended columnar pre-rules.
- *
- * @return
- */
- override def genExtendedColumnarTransformRules(): List[SparkSession => Rule[SparkPlan]] =
- List()
-
- /**
- * Generate extended columnar post-rules.
- *
- * @return
- */
- override def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] =
- List(spark => NativeWritePostRule(spark))
-
- /**
- * Generate extended Strategies.
- *
- * @return
- */
- override def genExtendedStrategies(): List[SparkSession => Strategy] =
- List()
-
- override def genStringTranslateTransformer(
- substraitExprName: String,
- srcExpr: ExpressionTransformer,
- matchingExpr: ExpressionTransformer,
- replaceExpr: ExpressionTransformer,
- original: StringTranslate): ExpressionTransformer = {
- CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr, replaceExpr, original)
- }
-
- override def genSizeExpressionTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: Size): ExpressionTransformer = {
- CHSizeExpressionTransformer(substraitExprName, child, original)
- }
-
- /** Generate an ExpressionTransformer to transform TruncTimestamp expression for CH. */
- override def genTruncTimestampTransformer(
- substraitExprName: String,
- format: ExpressionTransformer,
- timestamp: ExpressionTransformer,
- timeZoneId: Option[String],
- original: TruncTimestamp): ExpressionTransformer = {
- CHTruncTimestampTransformer(substraitExprName, format, timestamp, timeZoneId, original)
- }
-
- override def genPosExplodeTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: PosExplode,
- attributeSeq: Seq[Attribute]): ExpressionTransformer = {
- CHPosExplodeTransformer(substraitExprName, child, original, attributeSeq)
- }
-
- override def genRegexpReplaceTransformer(
- substraitExprName: String,
- children: Seq[ExpressionTransformer],
- expr: RegExpReplace): ExpressionTransformer = {
- CHRegExpReplaceTransformer(substraitExprName, children, expr)
- }
-
- override def createColumnarWriteFilesExec(
- child: SparkPlan,
- fileFormat: FileFormat,
- partitionColumns: Seq[Attribute],
- bucketSpec: Option[BucketSpec],
- options: Map[String, String],
- staticPartitions: TablePartitionSpec): WriteFilesExec = {
- throw new GlutenNotSupportException("ColumnarWriteFilesExec is not support in ch backend.")
- }
-
- /**
- * Define whether the join operator is fallback because of the join operator is not supported by
- * backend
- */
- override def joinFallback(
- JoinType: JoinType,
- leftOutputSet: AttributeSet,
- rightOutputSet: AttributeSet,
- condition: Option[Expression]): Boolean = {
- CHJoinValidateUtil.shouldFallback(JoinType, leftOutputSet, rightOutputSet, condition)
- }
-
- /** Generate window function node */
- override def genWindowFunctionsNode(
- windowExpression: Seq[NamedExpression],
- windowExpressionNodes: JList[WindowFunctionNode],
- originalInputAttributes: Seq[Attribute],
- args: JMap[String, JLong]): Unit = {
-
- windowExpression.map {
- windowExpr =>
- val aliasExpr = windowExpr.asInstanceOf[Alias]
- val columnName = s"${aliasExpr.name}_${aliasExpr.exprId.id}"
- val wExpression = aliasExpr.child.asInstanceOf[WindowExpression]
- wExpression.windowFunction match {
- case wf @ (RowNumber() | Rank(_) | DenseRank(_) | CumeDist() | PercentRank(_)) =>
- val aggWindowFunc = wf.asInstanceOf[AggregateWindowFunction]
- val frame = aggWindowFunc.frame.asInstanceOf[SpecifiedWindowFrame]
- val windowFunctionNode = ExpressionBuilder.makeWindowFunction(
- WindowFunctionsBuilder.create(args, aggWindowFunc).toInt,
- new JArrayList[ExpressionNode](),
- columnName,
- ConverterUtils.getTypeNode(aggWindowFunc.dataType, aggWindowFunc.nullable),
- WindowExecTransformer.getFrameBound(frame.upper),
- WindowExecTransformer.getFrameBound(frame.lower),
- frame.frameType.sql
- )
- windowExpressionNodes.add(windowFunctionNode)
- case aggExpression: AggregateExpression =>
- val frame = wExpression.windowSpec.frameSpecification.asInstanceOf[SpecifiedWindowFrame]
- val aggregateFunc = aggExpression.aggregateFunction
- val substraitAggFuncName = ExpressionMappings.expressionsMap.get(aggregateFunc.getClass)
- if (substraitAggFuncName.isEmpty) {
- throw new GlutenNotSupportException(s"Not currently supported: $aggregateFunc.")
- }
-
- val childrenNodeList = new JArrayList[ExpressionNode]()
- aggregateFunc.children.foreach(
- expr =>
- childrenNodeList.add(
- ExpressionConverter
- .replaceWithExpressionTransformer(expr, originalInputAttributes)
- .doTransform(args)))
-
- val windowFunctionNode = ExpressionBuilder.makeWindowFunction(
- AggregateFunctionsBuilder.create(args, aggExpression.aggregateFunction).toInt,
- childrenNodeList,
- columnName,
- ConverterUtils.getTypeNode(aggExpression.dataType, aggExpression.nullable),
- WindowExecTransformer.getFrameBound(frame.upper),
- WindowExecTransformer.getFrameBound(frame.lower),
- frame.frameType.sql
- )
- windowExpressionNodes.add(windowFunctionNode)
- case wf @ (Lead(_, _, _, _) | Lag(_, _, _, _)) =>
- val (offsetWf, frame) = wf match {
- case lead @ Lead(input, offset, default, ignoreNulls) =>
- // When the offset value of the lead is negative, will convert to lag function
- lead.offset match {
- case IntegerLiteral(value) if value < 0 =>
- val newWf = Lag(input, Literal(math.abs(value)), default, ignoreNulls)
- (newWf, newWf.frame.asInstanceOf[SpecifiedWindowFrame])
- case other => (lead, lead.frame.asInstanceOf[SpecifiedWindowFrame])
- }
- case lag @ Lag(input, offset, default, ignoreNulls) =>
- // When the offset value of the lag is negative, will convert to lead function
- lag.offset match {
- case IntegerLiteral(value) if value > 0 =>
- val newWf = Lead(input, Literal(value), default, ignoreNulls)
- (newWf, newWf.frame.asInstanceOf[SpecifiedWindowFrame])
- case other => (lag, lag.frame.asInstanceOf[SpecifiedWindowFrame])
- }
- }
-
- val childrenNodeList = new JArrayList[ExpressionNode]()
- childrenNodeList.add(
- ExpressionConverter
- .replaceWithExpressionTransformer(
- offsetWf.input,
- attributeSeq = originalInputAttributes)
- .doTransform(args))
- childrenNodeList.add(
- ExpressionConverter
- .replaceWithExpressionTransformer(
- offsetWf.offset,
- attributeSeq = originalInputAttributes)
- .doTransform(args))
- childrenNodeList.add(
- ExpressionConverter
- .replaceWithExpressionTransformer(
- offsetWf.default,
- attributeSeq = originalInputAttributes)
- .doTransform(args))
- val windowFunctionNode = ExpressionBuilder.makeWindowFunction(
- WindowFunctionsBuilder.create(args, offsetWf).toInt,
- childrenNodeList,
- columnName,
- ConverterUtils.getTypeNode(offsetWf.dataType, offsetWf.nullable),
- WindowExecTransformer.getFrameBound(frame.upper),
- WindowExecTransformer.getFrameBound(frame.lower),
- frame.frameType.sql
- )
- windowExpressionNodes.add(windowFunctionNode)
- case _ =>
- throw new GlutenNotSupportException(
- "unsupported window function type: " +
- wExpression.windowFunction)
- }
- }
- }
-
- /** Clickhouse Backend only supports part of filters for parquet. */
- override def postProcessPushDownFilter(
- extraFilters: Seq[Expression],
- sparkExecNode: LeafExecNode): Seq[Expression] = {
- // FIXME: DeltaMergeTreeFileFormat should not inherit from ParquetFileFormat.
- def isParquetFormat(fileFormat: FileFormat): Boolean = fileFormat match {
- case p: ParquetFileFormat if p.shortName().equals("parquet") => true
- case _ => false
- }
-
- // TODO: datasource v2 ?
- // TODO: Push down conditions with scalar subquery
- // For example, consider TPCH 22 'c_acctbal > (select avg(c_acctbal) from customer where ...)'.
- // Vanilla Spark only pushes down the Parquet Filter not Catalyst Filter, which can not get the
- // subquery result, while gluten pushes down the Catalyst Filter can benefit from this case.
- //
- // Let's make push down functionally same as vanilla Spark for now.
-
- sparkExecNode match {
- case fileSourceScan: FileSourceScanExec
- if isParquetFormat(fileSourceScan.relation.fileFormat) =>
- PushDownUtil.removeNotSupportPushDownFilters(
- fileSourceScan.conf,
- fileSourceScan.output,
- fileSourceScan.dataFilters)
- case _ => super.postProcessPushDownFilter(extraFilters, sparkExecNode)
- }
- }
-
- override def genGenerateTransformer(
- generator: Generator,
- requiredChildOutput: Seq[Attribute],
- outer: Boolean,
- generatorOutput: Seq[Attribute],
- child: SparkPlan
- ): GenerateExecTransformerBase = {
- CHGenerateExecTransformer(generator, requiredChildOutput, outer, generatorOutput, child)
- }
-
- override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = generate
-
- override def genPostProjectForGenerate(generate: GenerateExec): SparkPlan = generate
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHTransformerApi.scala
deleted file mode 100644
index 88d0e5c44..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHTransformerApi.scala
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.backendsapi.TransformerApi
-import io.glutenproject.execution.CHHashAggregateExecTransformer
-import io.glutenproject.expression.ConverterUtils
-import io.glutenproject.substrait.expression.{BooleanLiteralNode, ExpressionBuilder, ExpressionNode}
-import io.glutenproject.utils.{CHInputPartitionsUtil, ExpressionDocUtil}
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.read.InputPartition
-import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
-import org.apache.spark.sql.delta.files.TahoeFileIndex
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.aggregate.HashAggregateExec
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory}
-import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat
-import org.apache.spark.sql.types._
-import org.apache.spark.util.collection.BitSet
-
-import com.google.common.collect.Lists
-import com.google.protobuf.{Any, Message}
-
-import java.util
-
-class CHTransformerApi extends TransformerApi with Logging {
-
- /** Generate Seq[InputPartition] for FileSourceScanExecTransformer. */
- def genInputPartitionSeq(
- relation: HadoopFsRelation,
- selectedPartitions: Array[PartitionDirectory],
- output: Seq[Attribute],
- bucketedScan: Boolean,
- optionalBucketSet: Option[BitSet],
- optionalNumCoalescedBuckets: Option[Int],
- disableBucketedScan: Boolean): Seq[InputPartition] = {
- relation.location match {
- case index: TahoeFileIndex
- if relation.fileFormat
- .isInstanceOf[DeltaMergeTreeFileFormat] =>
- // Generate NativeMergeTreePartition for MergeTree
- ClickHouseTableV2
- .partsPartitions(
- index.deltaLog,
- relation,
- selectedPartitions,
- output,
- bucketedScan,
- optionalBucketSet,
- optionalNumCoalescedBuckets,
- disableBucketedScan
- )
- case _: TahoeFileIndex =>
- throw new UnsupportedOperationException("Does not support delta-parquet")
- case _ =>
- // Generate FilePartition for Parquet
- CHInputPartitionsUtil(
- relation,
- selectedPartitions,
- output,
- bucketedScan,
- optionalBucketSet,
- optionalNumCoalescedBuckets,
- disableBucketedScan).genInputPartitionSeq()
- }
- }
-
- override def postProcessNativeConfig(
- nativeConfMap: util.Map[String, String],
- backendPrefix: String): Unit = {
- val settingPrefix = backendPrefix + ".runtime_settings."
- if (nativeConfMap.getOrDefault("spark.memory.offHeap.enabled", "false").toBoolean) {
- val offHeapSize =
- nativeConfMap.getOrDefault("spark.gluten.memory.offHeap.size.in.bytes", "0").toLong
- if (offHeapSize > 0) {
- // Only set default max_bytes_before_external_sort for CH when it is not set explicitly.
- val sortSpillKey = settingPrefix + "max_bytes_before_external_sort";
- if (!nativeConfMap.containsKey(sortSpillKey)) {
- val sortSpillValue = offHeapSize * 0.5
- nativeConfMap.put(sortSpillKey, sortSpillValue.toLong.toString)
- }
-
- // Only set default max_bytes_before_external_group_by for CH when it is not set explicitly.
- val groupBySpillKey = settingPrefix + "max_bytes_before_external_group_by";
- if (!nativeConfMap.containsKey(groupBySpillKey)) {
- val groupBySpillValue = offHeapSize * 0.5
- nativeConfMap.put(groupBySpillKey, groupBySpillValue.toLong.toString)
- }
-
- val maxMemoryUsageKey = settingPrefix + "max_memory_usage";
- if (!nativeConfMap.containsKey(maxMemoryUsageKey)) {
- val maxMemoryUsageValue = offHeapSize
- nativeConfMap.put(maxMemoryUsageKey, maxMemoryUsageValue.toLong.toString)
- }
-
- // Only set default max_bytes_before_external_join for CH when join_algorithm is grace_hash
- val joinAlgorithmKey = settingPrefix + "join_algorithm";
- if (
- nativeConfMap.containsKey(joinAlgorithmKey) &&
- nativeConfMap.get(joinAlgorithmKey) == "grace_hash"
- ) {
- val joinSpillKey = settingPrefix + "max_bytes_in_join";
- if (!nativeConfMap.containsKey(joinSpillKey)) {
- val joinSpillValue = offHeapSize * 0.7
- nativeConfMap.put(joinSpillKey, joinSpillValue.toLong.toString)
- }
- }
- }
- }
-
- val injectConfig: (String, String) => Unit = (srcKey, dstKey) => {
- if (nativeConfMap.containsKey(srcKey) && !nativeConfMap.containsKey(dstKey)) {
- nativeConfMap.put(dstKey, nativeConfMap.get(srcKey))
- }
- }
-
- val hdfsConfigPrefix = backendPrefix + ".runtime_config.hdfs."
- injectConfig("spark.hadoop.input.connect.timeout", hdfsConfigPrefix + "input_connect_timeout")
- injectConfig("spark.hadoop.input.read.timeout", hdfsConfigPrefix + "input_read_timeout")
- injectConfig("spark.hadoop.input.write.timeout", hdfsConfigPrefix + "input_write_timeout")
- injectConfig(
- "spark.hadoop.dfs.client.log.severity",
- hdfsConfigPrefix + "dfs_client_log_severity")
-
- // TODO: set default to true when metrics could be collected
- // while ch query plan optimization is enabled.
- val planOptKey = settingPrefix + "query_plan_enable_optimizations"
- if (!nativeConfMap.containsKey(planOptKey)) {
- nativeConfMap.put(planOptKey, "false")
- }
-
- // Respect spark config spark.sql.orc.compression.codec for CH backend
- // TODO: consider compression or orc.compression in table options.
- val orcCompressionKey = settingPrefix + "output_format_orc_compression_method"
- if (!nativeConfMap.containsKey(orcCompressionKey)) {
- if (nativeConfMap.containsKey("spark.sql.orc.compression.codec")) {
- val compression = nativeConfMap.get("spark.sql.orc.compression.codec").toLowerCase()
- compression match {
- case "none" => nativeConfMap.put(orcCompressionKey, "none")
- case "uncompressed" => nativeConfMap.put(orcCompressionKey, "none")
- case "snappy" => nativeConfMap.put(orcCompressionKey, "snappy")
- case "zlib" => nativeConfMap.put(orcCompressionKey, "zlib")
- case "zstd" => nativeConfMap.put(orcCompressionKey, "zstd")
- case "lz4" => nativeConfMap.put(orcCompressionKey, "lz4")
- case _ =>
- throw new UnsupportedOperationException(s"Not supported ORC compression: $compression")
- }
- } else {
- nativeConfMap.put(orcCompressionKey, "snappy")
- }
- }
- }
-
- override def getSupportExpressionClassName: util.Set[String] = {
- ExpressionDocUtil.supportExpression()
- }
-
- override def getPlanOutput(plan: SparkPlan): Seq[Attribute] = {
- plan match {
- case hash: HashAggregateExec =>
- // when grouping expression has alias,
- // output name will be different from grouping expressions,
- // so using output attribute instead of grouping expression
- val groupingExpressions = hash.output.splitAt(hash.groupingExpressions.size)._1
- val aggResultAttributes = CHHashAggregateExecTransformer.getAggregateResultAttributes(
- groupingExpressions,
- hash.aggregateExpressions
- )
- if (aggResultAttributes.size == hash.output.size) {
- aggResultAttributes
- } else {
- var output = Seq.empty[Attribute]
- for (i <- hash.output.indices) {
- if (i < groupingExpressions.size) {
- output = output :+ aggResultAttributes(i)
- } else {
- output = output :+ hash.output(i)
- }
- }
- output
- }
- case _ =>
- plan.output
- }
-
- }
-
- override def createDateDiffParamList(
- start: ExpressionNode,
- end: ExpressionNode): Iterable[ExpressionNode] = {
- List(ExpressionBuilder.makeStringLiteral("day"), start, end)
- }
-
- override def createLikeParamList(
- left: ExpressionNode,
- right: ExpressionNode,
- escapeChar: ExpressionNode): Iterable[ExpressionNode] =
- List(left, right)
-
- override def createCheckOverflowExprNode(
- args: java.lang.Object,
- substraitExprName: String,
- childNode: ExpressionNode,
- dataType: DecimalType,
- nullable: Boolean,
- nullOnOverflow: Boolean): ExpressionNode = {
- val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]]
- val functionId = ExpressionBuilder.newScalarFunction(
- functionMap,
- ConverterUtils.makeFuncName(
- substraitExprName,
- Seq(dataType, BooleanType),
- ConverterUtils.FunctionConfig.OPT))
-
- // Just make a fake toType value, because native engine cannot accept datatype itself.
- val toTypeNodes =
- ExpressionBuilder.makeDecimalLiteral(new Decimal().set(0, dataType.precision, dataType.scale))
- val expressionNodes =
- Lists.newArrayList(childNode, new BooleanLiteralNode(nullOnOverflow), toTypeNodes)
- val typeNode = ConverterUtils.getTypeNode(dataType, nullable)
- ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode)
- }
-
- override def getNativePlanString(substraitPlan: Array[Byte], details: Boolean): String = {
- throw new UnsupportedOperationException("CH backend does not support this method")
- }
-
- override def packPBMessage(message: Message): Any = Any.pack(message)
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHValidatorApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHValidatorApi.scala
deleted file mode 100644
index cbf00c08b..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHValidatorApi.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.backendsapi.clickhouse
-
-import io.glutenproject.GlutenConfig
-import io.glutenproject.backendsapi.{BackendsApiManager, ValidatorApi}
-import io.glutenproject.expression.ExpressionConverter
-import io.glutenproject.substrait.SubstraitContext
-import io.glutenproject.substrait.expression.SelectionNode
-import io.glutenproject.substrait.plan.PlanNode
-import io.glutenproject.utils.CHExpressionUtil
-import io.glutenproject.validate.NativePlanValidationInfo
-import io.glutenproject.vectorized.CHNativeExpressionEvaluator
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.shuffle.utils.RangePartitionerBoundsGenerator
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning}
-import org.apache.spark.sql.delta.DeltaLogFileIndex
-import org.apache.spark.sql.execution.{CommandResultExec, FileSourceScanExec, RDDScanExec, SparkPlan}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
-
-class CHValidatorApi extends ValidatorApi with AdaptiveSparkPlanHelper with Logging {
-
- override def doNativeValidateWithFailureReason(plan: PlanNode): NativePlanValidationInfo = {
- val validator = new CHNativeExpressionEvaluator()
- if (validator.doValidate(plan.toProtobuf.toByteArray)) {
- new NativePlanValidationInfo(1, "")
- } else {
- new NativePlanValidationInfo(0, "CH native check failed.")
- }
- }
-
- /**
- * Validate expression for specific backend, including input type. If the expression isn't
- * implemented by the backend or it returns mismatched results with Vanilla Spark, it will fall
- * back to Vanilla Spark.
- *
- * @return
- * true by default
- */
- override def doExprValidate(substraitExprName: String, expr: Expression): Boolean = {
- CHExpressionUtil.CH_BLACKLIST_SCALAR_FUNCTION.get(substraitExprName) match {
- case Some(validator) =>
- return validator.doValidate(expr)
- case _ =>
- }
- CHExpressionUtil.CH_AGGREGATE_FUNC_BLACKLIST.get(substraitExprName) match {
- case Some(validator) =>
- return validator.doValidate(expr)
- case _ =>
- }
- true
- }
-
- /** Validate against a whole Spark plan, before being interpreted by Gluten. */
- override def doSparkPlanValidate(plan: SparkPlan): Boolean = {
- // TODO: Currently there are some fallback issues on CH backend when SparkPlan is
- // TODO: SerializeFromObjectExec, ObjectHashAggregateExec and V2CommandExec.
- // For example:
- // val tookTimeArr = Array(12, 23, 56, 100, 500, 20)
- // import spark.implicits._
- // val df = spark.sparkContext.parallelize(tookTimeArr.toSeq, 1).toDF("time")
- // df.summary().show(100, false)
-
- def includedDeltaOperator(scanExec: FileSourceScanExec): Boolean = {
- scanExec.relation.location.isInstanceOf[DeltaLogFileIndex]
- }
-
- val includedUnsupportedPlans = collect(plan) {
- // case s: SerializeFromObjectExec => true
- // case d: DeserializeToObjectExec => true
- // case o: ObjectHashAggregateExec => true
- case rddScanExec: RDDScanExec if rddScanExec.nodeName.contains("Delta Table State") => true
- case f: FileSourceScanExec if includedDeltaOperator(f) => true
- case v2CommandExec: V2CommandExec => true
- case commandResultExec: CommandResultExec => true
- }
-
- !includedUnsupportedPlans.contains(true)
- }
-
- /** Validate whether the compression method support splittable at clickhouse backend. */
- override def doCompressionSplittableValidate(compressionMethod: String): Boolean = {
- false
- }
-
- override def doColumnarShuffleExchangeExecValidate(
- outputPartitioning: Partitioning,
- child: SparkPlan): Option[String] = {
- val outputAttributes = child.output
- // check repartition expression
- val substraitContext = new SubstraitContext
- outputPartitioning match {
- case HashPartitioning(exprs, _) =>
- val allSelectionNodes = exprs.forall {
- expr =>
- val node = ExpressionConverter
- .replaceWithExpressionTransformer(expr, outputAttributes)
- .doTransform(substraitContext.registeredFunction)
- node.isInstanceOf[SelectionNode]
- }
- if (
- allSelectionNodes ||
- BackendsApiManager.getSettings.supportShuffleWithProject(outputPartitioning, child)
- ) {
- None
- } else {
- Some("expressions are not supported in HashPartitioning")
- }
- case rangePartitoning: RangePartitioning =>
- if (
- GlutenConfig.getConf.enableColumnarSort &&
- RangePartitionerBoundsGenerator.supportedOrderings(rangePartitoning, child)
- ) {
- None
- } else {
- Some("do not support range partitioning columnar sort")
- }
- case _ => None
- }
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHBroadcastBuildSideCache.scala b/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHBroadcastBuildSideCache.scala
deleted file mode 100644
index df9733041..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHBroadcastBuildSideCache.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.execution
-
-import io.glutenproject.backendsapi.clickhouse.CHBackendSettings
-import io.glutenproject.vectorized.StorageJoinBuilder
-
-import org.apache.spark.SparkEnv
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.execution.joins.{BuildSideRelation, ClickHouseBuildSideRelation}
-
-import com.github.benmanes.caffeine.cache.{Cache, Caffeine, RemovalCause, RemovalListener}
-
-import java.util.concurrent.TimeUnit
-
-case class BroadcastHashTable(pointer: Long, relation: ClickHouseBuildSideRelation)
-
-/**
- * `CHBroadcastBuildSideCache` is used for controlling to build bhj hash table once.
- *
- * The complicated part is due to reuse exchange, where multiple BHJ IDs correspond to a
- * `ClickHouseBuildSideRelation`.
- */
-object CHBroadcastBuildSideCache extends Logging with RemovalListener[String, BroadcastHashTable] {
-
- private def threadLog(msg: => String): Unit =
- logDebug(s"Thread: ${Thread.currentThread().getId} -- $msg")
-
- private lazy val expiredTime = SparkEnv.get.conf.getLong(
- CHBackendSettings.GLUTEN_CLICKHOUSE_BROADCAST_CACHE_EXPIRED_TIME,
- CHBackendSettings.GLUTEN_CLICKHOUSE_BROADCAST_CACHE_EXPIRED_TIME_DEFAULT
- )
-
- // Use for controlling to build bhj hash table once.
- // key: hashtable id, value is hashtable backend pointer(long to string).
- private val buildSideRelationCache: Cache[String, BroadcastHashTable] =
- Caffeine.newBuilder
- .expireAfterAccess(expiredTime, TimeUnit.SECONDS)
- .removalListener(this)
- .build[String, BroadcastHashTable]()
-
- def getOrBuildBroadcastHashTable(
- broadcast: Broadcast[BuildSideRelation],
- broadCastContext: BroadCastHashJoinContext): BroadcastHashTable = {
-
- buildSideRelationCache
- .get(
- broadCastContext.buildHashTableId,
- (broadcast_id: String) => {
- val (pointer, relation) =
- broadcast.value
- .asInstanceOf[ClickHouseBuildSideRelation]
- .buildHashTable(broadCastContext)
- threadLog(s"create bhj $broadcast_id = 0x${pointer.toHexString}")
- BroadcastHashTable(pointer, relation)
- }
- )
- }
-
- /** This is callback from c++ backend. */
- def get(broadcastHashtableId: String): Long =
- Option(buildSideRelationCache.getIfPresent(broadcastHashtableId))
- .map(_.pointer)
- .getOrElse(0)
-
- def invalidateBroadcastHashtable(broadcastHashtableId: String): Unit = {
- // Cleanup operations on the backend are idempotent.
- buildSideRelationCache.invalidate(broadcastHashtableId)
- }
-
- /** Only used in UT. */
- def size(): Long = buildSideRelationCache.estimatedSize()
-
- def cleanAll(): Unit = buildSideRelationCache.invalidateAll()
-
- override def onRemoval(key: String, value: BroadcastHashTable, cause: RemovalCause): Unit = {
- threadLog(s"remove bhj $key = 0x${value.pointer.toHexString}")
- if (value.relation != null) {
- value.relation.reset()
- }
- StorageJoinBuilder.nativeCleanBuildHashTable(key, value.pointer)
- }
-}
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHFilterExecTransformer.scala b/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHFilterExecTransformer.scala
deleted file mode 100644
index 9b4d55a1d..000000000
--- a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHFilterExecTransformer.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.glutenproject.execution
-
-import org.apache.spark.sql.catalyst.expressions.{And, Expression}
-import org.apache.spark.sql.execution.SparkPlan
-
-case class CHFilterExecTransformer(condition: Expression, child: SparkPlan)
- extends FilterExecTransformerBase(condition, child) {
- override protected def getRemainingCondition: Expression = {
- val scanFilters = child match {
- // Get the filters including the manually pushed down ones.
... 220367 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org