You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ya...@apache.org on 2024/09/05 02:06:00 UTC
(incubator-gluten) branch main updated: [GLUTEN-5880][CORE] Ignore fallback for ColumnarWriteFilesExec children (#7113)
This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8b5e3ce57 [GLUTEN-5880][CORE] Ignore fallback for ColumnarWriteFilesExec children (#7113)
8b5e3ce57 is described below
commit 8b5e3ce5717486fe4fdbbfa202aff80ee9a3207a
Author: Zhen Wang <64...@qq.com>
AuthorDate: Thu Sep 5 10:05:57 2024 +0800
[GLUTEN-5880][CORE] Ignore fallback for ColumnarWriteFilesExec children (#7113)
---
.../apache/spark/sql/execution/ColumnarWriteFilesExec.scala | 2 +-
.../org/apache/spark/sql/execution/GlutenExplainUtils.scala | 4 ++++
.../org/apache/spark/sql/execution/GlutenImplicits.scala | 4 ++++
.../org/apache/spark/sql/sources/GlutenInsertSuite.scala | 12 ++++++++++--
.../org/apache/spark/sql/sources/GlutenInsertSuite.scala | 12 ++++++++++--
5 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
index f8f596dce..92dd74796 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
+++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
@@ -150,7 +150,7 @@ object ColumnarWriteFilesExec {
staticPartitions)
}
- private case class NoopLeaf() extends LeafExecNode {
+ case class NoopLeaf() extends LeafExecNode {
override protected def doExecute(): RDD[InternalRow] =
throw new GlutenException(s"$nodeName does not support doExecute")
override def output: Seq[Attribute] = Seq.empty
diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala
index 338136b6d..43b74c883 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala
+++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala
@@ -24,9 +24,11 @@ import org.apache.gluten.utils.PlanUtil
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression}
import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.execution.ColumnarWriteFilesExec.NoopLeaf
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, QueryStageExec}
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
+import org.apache.spark.sql.execution.datasources.WriteFilesExec
import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec}
@@ -90,6 +92,8 @@ object GlutenExplainUtils extends AdaptiveSparkPlanHelper {
case _: ColumnarToRowTransition =>
case _: RowToColumnarTransition =>
case _: ReusedExchangeExec =>
+ case _: NoopLeaf =>
+ case w: WriteFilesExec if w.child.isInstanceOf[NoopLeaf] =>
case sub: AdaptiveSparkPlanExec if sub.isSubquery => collect(sub.executedPlan)
case _: AdaptiveSparkPlanExec =>
case p: QueryStageExec => collect(p.plan)
diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala
index 2e2af6517..53b1e861b 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala
+++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala
@@ -24,10 +24,12 @@ import org.apache.spark.sql.{AnalysisException, Dataset, SparkSession}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.{CommandResult, LogicalPlan}
import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
+import org.apache.spark.sql.execution.ColumnarWriteFilesExec.NoopLeaf
import org.apache.spark.sql.execution.GlutenExplainUtils._
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec}
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
+import org.apache.spark.sql.execution.datasources.WriteFilesExec
import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.internal.SQLConf
@@ -112,6 +114,8 @@ object GlutenImplicits {
case _: ColumnarToRowTransition =>
case _: RowToColumnarTransition =>
case p: ReusedExchangeExec =>
+ case _: NoopLeaf =>
+ case w: WriteFilesExec if w.child.isInstanceOf[NoopLeaf] =>
case p: AdaptiveSparkPlanExec if isFinalAdaptivePlan(p) =>
collect(p.executedPlan)
case p: AdaptiveSparkPlanExec =>
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index 3c334511a..74c4df197 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.executor.OutputMetrics
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.{ColumnarWriteFilesExec, CommandResultExec, QueryExecution}
+import org.apache.spark.sql.execution.{ColumnarWriteFilesExec, CommandResultExec, GlutenImplicits, QueryExecution}
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.command.DataWritingCommandExec
import org.apache.spark.sql.execution.metric.SQLMetric
@@ -70,8 +70,9 @@ class GlutenInsertSuite
}
testGluten("insert partition table") {
- withTable("pt") {
+ withTable("pt", "pt2") {
spark.sql("CREATE TABLE pt (c1 int, c2 string) USING PARQUET PARTITIONED BY (pt string)")
+ spark.sql("CREATE TABLE pt2 (c1 int, c2 string) USING PARQUET PARTITIONED BY (pt string)")
var taskMetrics: OutputMetrics = null
val taskListener = new SparkListener {
@@ -111,6 +112,13 @@ class GlutenInsertSuite
spark.sparkContext.removeSparkListener(taskListener)
spark.listenerManager.unregister(queryListener)
}
+
+ // check no fallback nodes
+ val df2 = spark.sql("INSERT INTO TABLE pt2 SELECT * FROM pt")
+ checkAndGetWriteFiles(df2)
+ val fallbackSummary = GlutenImplicits
+ .collectQueryExecutionFallbackSummary(spark, df2.queryExecution)
+ assert(fallbackSummary.numFallbackNodes == 0)
}
}
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index 3d9d8842f..1cb905e10 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -25,7 +25,7 @@ import org.apache.spark.executor.OutputMetrics
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.{CommandResultExec, QueryExecution, SparkPlan}
+import org.apache.spark.sql.execution.{CommandResultExec, GlutenImplicits, QueryExecution, SparkPlan}
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.command.DataWritingCommandExec
import org.apache.spark.sql.execution.metric.SQLMetric
@@ -72,8 +72,9 @@ class GlutenInsertSuite
}
testGluten("insert partition table") {
- withTable("pt") {
+ withTable("pt", "pt2") {
spark.sql("CREATE TABLE pt (c1 int, c2 string) USING PARQUET PARTITIONED BY (pt string)")
+ spark.sql("CREATE TABLE pt2 (c1 int, c2 string) USING PARQUET PARTITIONED BY (pt string)")
var taskMetrics: OutputMetrics = null
val taskListener = new SparkListener {
@@ -113,6 +114,13 @@ class GlutenInsertSuite
spark.sparkContext.removeSparkListener(taskListener)
spark.listenerManager.unregister(queryListener)
}
+
+ // check no fallback nodes
+ val df2 = spark.sql("INSERT INTO TABLE pt2 SELECT * FROM pt")
+ checkWriteFilesAndGetChild(df2)
+ val fallbackSummary = GlutenImplicits
+ .collectQueryExecutionFallbackSummary(spark, df2.queryExecution)
+ assert(fallbackSummary.numFallbackNodes == 0)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org