(incubator-gluten) branch main updated: [CORE] Following #6745, append some minor code cleanups (#6788)

[ho...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new e132bbfcc [CORE] Following #6745, append some minor code cleanups (#6788)
e132bbfcc is described below

commit e132bbfcc655c474ed561f5e0df4cc0a883bdf85
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Tue Aug 13 13:18:53 2024 +0800

    [CORE] Following #6745, append some minor code cleanups (#6788)
---
 .../sql/execution/ColumnarWriteFilesExec.scala     | 47 +++++++++++++---------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
index 9c54653d9..f8f596dce 100644
--- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet}
 import org.apache.spark.sql.connector.write.WriterCommitMessage
 import org.apache.spark.sql.execution.datasources._
+import org.apache.spark.sql.vectorized.ColumnarBatch
 
 // The class inherits from "BinaryExecNode" instead of "UnaryExecNode" because
 // we need to expose a dummy child (as right child) with type "WriteFilesExec" to let Spark
@@ -43,6 +44,8 @@ abstract class ColumnarWriteFilesExec protected (
     override val right: SparkPlan)
   extends BinaryExecNode
   with GlutenPlan
+  with KnownChildrenConventions
+  with KnownRowType
   with ColumnarWriteFilesExec.ExecuteWriteCompatible {
 
   val child: SparkPlan = left
@@ -51,10 +54,32 @@ abstract class ColumnarWriteFilesExec protected (
 
   override def supportsColumnar: Boolean = true
 
+  override def requiredChildrenConventions(): Seq[ConventionReq] = {
+    List(ConventionReq.backendBatch)
+  }
+
+  /**
+   * Mark the plan node as outputting both row-based and columnar data. Then we could avoid
+   * unnecessary transitions from being added on the exit side of the node.
+   *
+   * This is feasible based on the assumption that the node doesn't actually involve in either row
+   * processing or columnar processing. It's true because Spark only calls `doExecuteWrite` of the
+   * object.
+   *
+   * Since https://github.com/apache/incubator-gluten/pull/6745.
+   */
+  override def rowType(): RowType = {
+    RowType.VanillaRow
+  }
+
   override def output: Seq[Attribute] = Seq.empty
 
   override protected def doExecute(): RDD[InternalRow] = {
-    throw new GlutenException(s"$nodeName does not support doExecute")
+    throw new GlutenException(s"$nodeName does not implement #doExecute")
+  }
+
+  override protected def doExecuteColumnar(): RDD[ColumnarBatch] = {
+    throw new GlutenException(s"$nodeName does not implement #doExecuteColumnar")
   }
 
   /** Fallback to use vanilla Spark write files to generate an empty file for metadata only. */
@@ -84,7 +109,7 @@ abstract class ColumnarWriteFilesExec protected (
 
   /** We need this to avoid compiler error. */
   override def doExecuteWrite(writeFilesSpec: WriteFilesSpec): RDD[WriterCommitMessage] = {
-    super.doExecuteWrite(writeFilesSpec)
+    throw new GlutenException(s"$nodeName does not implement #doExecuteWrite")
   }
 }
 
@@ -131,24 +156,10 @@ object ColumnarWriteFilesExec {
     override def output: Seq[Attribute] = Seq.empty
   }
 
-  /**
-   * ColumnarWriteFilesExec neither output Row nor columnar data. We output both row and columnar to
-   * avoid c2r and r2c transitions. Please note, [[GlutenPlan]] already implement batchType()
-   */
-  sealed trait ExecuteWriteCompatible extends KnownChildrenConventions with KnownRowType {
+  sealed trait ExecuteWriteCompatible {
     // To be compatible with Spark (version < 3.4)
     protected def doExecuteWrite(writeFilesSpec: WriteFilesSpec): RDD[WriterCommitMessage] = {
-      throw new GlutenException(
-        s"Internal Error ${this.getClass} has write support" +
-          s" mismatch:\n${this}")
-    }
-
-    override def requiredChildrenConventions(): Seq[ConventionReq] = {
-      List(ConventionReq.backendBatch)
-    }
-
-    override def rowType(): RowType = {
-      RowType.VanillaRow
+      throw new GlutenException("Illegal state: The method is not expected to be called")
     }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org