You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by zh...@apache.org on 2024/11/01 02:05:46 UTC
(incubator-gluten) branch main updated: [GLUTEN-7670][CH] Fix enable 'files.per.partition.threshold' bug (#7758)
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new dc65d77d92 [GLUTEN-7670][CH] Fix enable 'files.per.partition.threshold' bug (#7758)
dc65d77d92 is described below
commit dc65d77d929785e8bce4b7c572281db2c3028d02
Author: Shuai li <lo...@live.cn>
AuthorDate: Fri Nov 1 10:05:42 2024 +0800
[GLUTEN-7670][CH] Fix enable 'files.per.partition.threshold' bug (#7758)
[CH] Fix enable 'files.per.partition.threshold' bug
---
.../utils/MergeTreePartsPartitionsUtil.scala | 9 ++++++++-
.../GlutenClickHouseMergeTreeWriteSuite.scala | 23 ++++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala
index 9347d8679b..cc04e9092d 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala
@@ -232,7 +232,14 @@ object MergeTreePartsPartitionsUtil extends Logging {
val splitFiles = selectRanges
.map {
part =>
- MergeTreePartSplit(part.name, part.dirName, part.targetNode, 0, part.marks, part.size)
+ MergeTreePartSplit(
+ part.name,
+ part.dirName,
+ part.targetNode,
+ part.start,
+ part.marks,
+ part.size
+ )
}
genInputPartitionSeqByFileCnt(
engine,
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala
index 0959832949..c1210c5fba 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala
@@ -1805,6 +1805,29 @@ class GlutenClickHouseMergeTreeWriteSuite
}
}
})
+
+ // GLUTEN-7670: fix enable 'files.per.partition.threshold'
+ withSQLConf(
+ CHConf.runtimeSettings("enabled_driver_filter_mergetree_index") -> "true",
+ CHConf.prefixOf("files.per.partition.threshold") -> "10"
+ ) {
+ runTPCHQueryBySQL(6, sqlStr) {
+ df =>
+ val scanExec = collect(df.queryExecution.executedPlan) {
+ case f: FileSourceScanExecTransformer => f
+ }
+ assertResult(1)(scanExec.size)
+
+ val mergetreeScan = scanExec.head
+ assert(mergetreeScan.nodeName.startsWith("Scan mergetree"))
+
+ val plans = collect(df.queryExecution.executedPlan) {
+ case scanExec: BasicScanExecTransformer => scanExec
+ }
+ assertResult(1)(plans.size)
+ assertResult(1)(plans.head.getSplitInfos(null).size)
+ }
+ }
}
test("test mergetree with primary keys filter pruning by driver with bucket") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org