You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ph...@apache.org on 2024/12/09 12:51:57 UTC
(incubator-gluten) branch main updated: [CORE] Fix performance issue when allScanPartitions size is very large (#8126)
This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 72fc04e20d [CORE] Fix performance issue when allScanPartitions size is very large (#8126)
72fc04e20d is described below
commit 72fc04e20d3dce09365bc5d5e40476f341c4620b
Author: WangGuangxin <wa...@gmail.com>
AuthorDate: Mon Dec 9 20:51:53 2024 +0800
[CORE] Fix performance issue when allScanPartitions size is very large (#8126)
---
.../main/scala/org/apache/gluten/execution/WholeStageTransformer.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
index dbfc11c136..a752f33ef6 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
+++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
@@ -373,7 +373,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f
* care of SCAN there won't be any other RDD for SCAN. As a result, genFirstStageIterator
* rather than genFinalStageIterator will be invoked
*/
- val allScanPartitions = basicScanExecTransformers.map(_.getPartitions)
+ val allScanPartitions = basicScanExecTransformers.map(_.getPartitions.toIndexedSeq)
val allScanSplitInfos =
getSplitInfosFromPartitions(basicScanExecTransformers, allScanPartitions)
if (GlutenConfig.getConf.enableHdfsViewfs) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org