You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/07/05 00:26:23 UTC

(spark) branch master updated: [SPARK-52632][SQL] Pretty display V2 write plan nodes

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new c2dd021492fa [SPARK-52632][SQL] Pretty display V2 write plan nodes
c2dd021492fa is described below

commit c2dd021492fa0e7d6546183fc3c6b03c2c24c291
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Jul 4 17:26:10 2025 -0700

    [SPARK-52632][SQL] Pretty display V2 write plan nodes
    
    ### What changes were proposed in this pull request?
    
    Pretty display V2 write plan nodes by overriding `def stringArgs`.
    
    ### Why are the changes needed?
    
    Better UX for UI display and `EXPLAIN` output
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, change affects UI display and `EXPLAIN` output for V2 write cases.
    
    ### How was this patch tested?
    
    Use AppendData as an example.
    
    Before
    <img width="1721" alt="Xnip2025-07-01_16-39-59" src="https://github.com/user-attachments/assets/9dcab5e2-c768-45a3-8c0b-f7059c8c1389" />
    After
    <img width="1712" alt="Xnip2025-07-01_16-30-00" src="https://github.com/user-attachments/assets/fd3fdd7c-da5e-49be-b801-f3debde02808" />
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #51332 from pan3793/SPARK-52632.
    
    Authored-by: Cheng Pan <ch...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../spark/sql/execution/datasources/noop/NoopDataSource.scala     | 5 +++++
 .../sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala    | 8 ++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index a662ea3b8d2d..c44c5f780a6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStr
 import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
 
 /**
  * This is no-op datasource. It does not do anything besides consuming its input.
@@ -48,6 +49,7 @@ private[noop] object NoopTable extends Table with SupportsWrite {
       TableCapability.TRUNCATE,
       TableCapability.ACCEPT_ANY_SCHEMA)
   }
+  override def toString: String = Utils.getFormattedClassName(this)
 }
 
 private[noop] object NoopWriteBuilder extends WriteBuilder
@@ -59,6 +61,7 @@ private[noop] object NoopWriteBuilder extends WriteBuilder
 private[noop] object NoopWrite extends Write {
   override def toBatch: BatchWrite = NoopBatchWrite
   override def toStreaming: StreamingWrite = NoopStreamingWrite
+  override def toString: String = Utils.getFormattedClassName(this)
 }
 
 private[noop] object NoopBatchWrite extends BatchWrite {
@@ -67,6 +70,7 @@ private[noop] object NoopBatchWrite extends BatchWrite {
   override def useCommitCoordinator(): Boolean = false
   override def commit(messages: Array[WriterCommitMessage]): Unit = {}
   override def abort(messages: Array[WriterCommitMessage]): Unit = {}
+  override def toString: String = Utils.getFormattedClassName(this)
 }
 
 private[noop] object NoopWriterFactory extends DataWriterFactory {
@@ -86,6 +90,7 @@ private[noop] object NoopStreamingWrite extends StreamingWrite {
   override def useCommitCoordinator(): Boolean = false
   override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
   override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
+  override def toString: String = Utils.getFormattedClassName(this)
 }
 
 private[noop] object NoopStreamingDataWriterFactory extends StreamingDataWriterFactory {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index 4436c6b24f7c..193662718ce4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -303,8 +303,6 @@ case class ReplaceDataExec(
     projections: ReplaceDataProjections,
     write: Write) extends V2ExistingTableWriteExec {
 
-  override val stringArgs: Iterator[Any] = Iterator(query, write)
-
   override def writingTask: WritingSparkTask[_] = {
     projections match {
       case ReplaceDataProjections(dataProj, Some(metadataProj)) =>
@@ -328,8 +326,6 @@ case class WriteDeltaExec(
     projections: WriteDeltaProjections,
     write: DeltaWrite) extends V2ExistingTableWriteExec {
 
-  override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
-
   override lazy val writingTask: WritingSparkTask[_] = {
     if (projections.metadataProjection.isDefined) {
       DeltaWithMetadataWritingSparkTask(projections)
@@ -349,6 +345,8 @@ case class WriteToDataSourceV2Exec(
     query: SparkPlan,
     writeMetrics: Seq[CustomMetric]) extends V2TableWriteExec {
 
+  override val stringArgs: Iterator[Any] = Iterator(batchWrite, query)
+
   override val customMetrics: Map[String, SQLMetric] = writeMetrics.map { customMetric =>
     customMetric.name() -> SQLMetrics.createV2CustomMetric(sparkContext, customMetric)
   }.toMap
@@ -367,6 +365,8 @@ trait V2ExistingTableWriteExec extends V2TableWriteExec {
   def refreshCache: () => Unit
   def write: Write
 
+  override val stringArgs: Iterator[Any] = Iterator(query, write)
+
   override val customMetrics: Map[String, SQLMetric] =
     write.supportedCustomMetrics().map { customMetric =>
       customMetric.name() -> SQLMetrics.createV2CustomMetric(sparkContext, customMetric)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org