You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/05/25 05:57:02 UTC
(spark) 01/02: [SPARK-56700][SS] Make DataStreamReader.name public
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
commit bdd5fdfde0da074026c58e761071220f624cda20
Author: ericm-db <er...@databricks.com>
AuthorDate: Tue May 5 22:10:05 2026 -0700
[SPARK-56700][SS] Make DataStreamReader.name public
Remove the `private[sql]` access modifier from `DataStreamReader.name` and add the method as a public abstract API to the `DataStreamReader` base class.
- Added abstract `def name(sourceName: String): this.type` to the API base class (`sql/api/.../DataStreamReader.scala`)
- Changed both classic and connect implementations from `private[sql] def name` to `override def name`
- Moved Scaladoc to the base class; implementations use `inheritdoc`
The `name` method was introduced in SPARK-56453 as `private[sql]` while the API was being finalized. Now that the feature is ready, making it public allows users to assign names to streaming sources for stable checkpoint metadata and source evolution.
Yes. `DataStreamReader.name(sourceName)` is now a public `Experimental` API available to all users. Previously it was package-private to `org.apache.spark.sql`.
Existing tests cover the `name` functionality. This change only modifies the access level.
Generated-by: Claude Code (Claude Opus 4.6)
Closes #55651 from ericm-db/datastreamreader-name-public.
Authored-by: ericm-db <er...@databricks.com>
Signed-off-by: Anish Shrigondekar <an...@databricks.com>
(cherry picked from commit 0af3d42c2c35cc6e7124eb1382aa240ecdf0f711)
---
project/MimaExcludes.scala | 2 ++
.../org/apache/spark/sql/streaming/DataStreamReader.scala | 13 ++++++++++++-
.../org/apache/spark/sql/connect/DataStreamReader.scala | 11 ++---------
.../org/apache/spark/sql/classic/DataStreamReader.scala | 10 ++--------
4 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index e92ef67080fb..e3eb166bcd8a 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -56,6 +56,8 @@ object MimaExcludes {
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.status.api.v1.ApplicationAttemptInfo$"),
// [SPARK-56330][CORE] Add TaskInterruptListener to TaskContext for interrupt notifications
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.addTaskInterruptListener"),
+ // [SPARK-56700][SS] Make DataStreamReader.name public
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.streaming.DataStreamReader.name"),
// [SPARK-34591][ML] Add pruneTree parameter to Strategy
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.tree.configuration.Strategy.this"),
// [SPARK-56395][SQL] Add NEAREST BY top-K ranking join
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 726b10d6416a..d4d8f162485f 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -18,7 +18,7 @@ package org.apache.spark.sql.streaming
import scala.jdk.CollectionConverters._
-import org.apache.spark.annotation.Evolving
+import org.apache.spark.annotation.{Evolving, Experimental}
import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, Encoders}
import org.apache.spark.sql.types.StructType
@@ -103,6 +103,17 @@ abstract class DataStreamReader {
this
}
+ /**
+ * Specifies a name for the streaming source. This name is used to identify the source in
+ * checkpoint metadata and enables stable checkpoint locations for source evolution.
+ *
+ * @param sourceName
+ * the name to assign to this streaming source
+ * @since 4.2.0
+ */
+ @Experimental
+ def name(sourceName: String): this.type
+
/**
* Loads input data stream in as a `DataFrame`, for data streams that don't require a path (e.g.
* external key-value stores).
diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataStreamReader.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataStreamReader.scala
index 2f8040273995..50c0de85e0cb 100644
--- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataStreamReader.scala
+++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataStreamReader.scala
@@ -75,16 +75,9 @@ final class DataStreamReader private[sql] (sparkSession: SparkSession)
this
}
- /**
- * Specifies a name for the streaming source. This name is used to identify the source in
- * checkpoint metadata and enables stable checkpoint locations for source evolution.
- *
- * @param sourceName
- * the name to assign to this streaming source
- * @since 4.2.0
- */
+ /** @inheritdoc */
@Experimental
- private[sql] def name(sourceName: String): this.type = {
+ override def name(sourceName: String): this.type = {
validateSourceName(sourceName)
sourceBuilder.setSourceName(sourceName)
this
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/DataStreamReader.scala
index 632f46acbe5b..c8df93768808 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/DataStreamReader.scala
@@ -67,15 +67,9 @@ final class DataStreamReader private[sql](sparkSession: SparkSession)
this
}
- /**
- * Specifies a name for the streaming source. This name is used to identify the source
- * in checkpoint metadata and enables stable checkpoint locations for source evolution.
- *
- * @param sourceName the name to assign to this streaming source
- * @since 4.2.0
- */
+ /** @inheritdoc */
@Experimental
- private[sql] def name(sourceName: String): this.type = {
+ override def name(sourceName: String): this.type = {
validateSourceName(sourceName)
this.userProvidedSourceName = Option(sourceName)
this
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org