You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/07/05 00:26:23 UTC
(spark) branch master updated: [SPARK-52632][SQL] Pretty display V2 write plan nodes
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c2dd021492fa [SPARK-52632][SQL] Pretty display V2 write plan nodes
c2dd021492fa is described below
commit c2dd021492fa0e7d6546183fc3c6b03c2c24c291
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Jul 4 17:26:10 2025 -0700
[SPARK-52632][SQL] Pretty display V2 write plan nodes
### What changes were proposed in this pull request?
Pretty display V2 write plan nodes by overriding `def stringArgs`.
### Why are the changes needed?
Better UX for UI display and `EXPLAIN` output
### Does this PR introduce _any_ user-facing change?
Yes, change affects UI display and `EXPLAIN` output for V2 write cases.
### How was this patch tested?
Use AppendData as an example.
Before
<img width="1721" alt="Xnip2025-07-01_16-39-59" src="https://github.com/user-attachments/assets/9dcab5e2-c768-45a3-8c0b-f7059c8c1389" />
After
<img width="1712" alt="Xnip2025-07-01_16-30-00" src="https://github.com/user-attachments/assets/fd3fdd7c-da5e-49be-b801-f3debde02808" />
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51332 from pan3793/SPARK-52632.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/sql/execution/datasources/noop/NoopDataSource.scala | 5 +++++
.../sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala | 8 ++++----
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index a662ea3b8d2d..c44c5f780a6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStr
import org.apache.spark.sql.sources.DataSourceRegister
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
/**
* This is no-op datasource. It does not do anything besides consuming its input.
@@ -48,6 +49,7 @@ private[noop] object NoopTable extends Table with SupportsWrite {
TableCapability.TRUNCATE,
TableCapability.ACCEPT_ANY_SCHEMA)
}
+ override def toString: String = Utils.getFormattedClassName(this)
}
private[noop] object NoopWriteBuilder extends WriteBuilder
@@ -59,6 +61,7 @@ private[noop] object NoopWriteBuilder extends WriteBuilder
private[noop] object NoopWrite extends Write {
override def toBatch: BatchWrite = NoopBatchWrite
override def toStreaming: StreamingWrite = NoopStreamingWrite
+ override def toString: String = Utils.getFormattedClassName(this)
}
private[noop] object NoopBatchWrite extends BatchWrite {
@@ -67,6 +70,7 @@ private[noop] object NoopBatchWrite extends BatchWrite {
override def useCommitCoordinator(): Boolean = false
override def commit(messages: Array[WriterCommitMessage]): Unit = {}
override def abort(messages: Array[WriterCommitMessage]): Unit = {}
+ override def toString: String = Utils.getFormattedClassName(this)
}
private[noop] object NoopWriterFactory extends DataWriterFactory {
@@ -86,6 +90,7 @@ private[noop] object NoopStreamingWrite extends StreamingWrite {
override def useCommitCoordinator(): Boolean = false
override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
+ override def toString: String = Utils.getFormattedClassName(this)
}
private[noop] object NoopStreamingDataWriterFactory extends StreamingDataWriterFactory {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index 4436c6b24f7c..193662718ce4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -303,8 +303,6 @@ case class ReplaceDataExec(
projections: ReplaceDataProjections,
write: Write) extends V2ExistingTableWriteExec {
- override val stringArgs: Iterator[Any] = Iterator(query, write)
-
override def writingTask: WritingSparkTask[_] = {
projections match {
case ReplaceDataProjections(dataProj, Some(metadataProj)) =>
@@ -328,8 +326,6 @@ case class WriteDeltaExec(
projections: WriteDeltaProjections,
write: DeltaWrite) extends V2ExistingTableWriteExec {
- override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
-
override lazy val writingTask: WritingSparkTask[_] = {
if (projections.metadataProjection.isDefined) {
DeltaWithMetadataWritingSparkTask(projections)
@@ -349,6 +345,8 @@ case class WriteToDataSourceV2Exec(
query: SparkPlan,
writeMetrics: Seq[CustomMetric]) extends V2TableWriteExec {
+ override val stringArgs: Iterator[Any] = Iterator(batchWrite, query)
+
override val customMetrics: Map[String, SQLMetric] = writeMetrics.map { customMetric =>
customMetric.name() -> SQLMetrics.createV2CustomMetric(sparkContext, customMetric)
}.toMap
@@ -367,6 +365,8 @@ trait V2ExistingTableWriteExec extends V2TableWriteExec {
def refreshCache: () => Unit
def write: Write
+ override val stringArgs: Iterator[Any] = Iterator(query, write)
+
override val customMetrics: Map[String, SQLMetric] =
write.supportedCustomMetrics().map { customMetric =>
customMetric.name() -> SQLMetrics.createV2CustomMetric(sparkContext, customMetric)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org