You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/04/13 15:21:19 UTC
(spark) branch master updated: [SPARK-56457][SQL] Fix Avro V2 formatName to match V1 FileFormat.toString
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6c9096e2b79c [SPARK-56457][SQL] Fix Avro V2 formatName to match V1 FileFormat.toString
6c9096e2b79c is described below
commit 6c9096e2b79c7001facbd00c363cf16e14b47447
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Mon Apr 13 08:21:10 2026 -0700
[SPARK-56457][SQL] Fix Avro V2 formatName to match V1 FileFormat.toString
### What changes were proposed in this pull request?
Fix `AvroTable.formatName` from `"AVRO"` to `"Avro"` to match the V1 `AvroFileFormat.toString` value. Add regression tests verifying V1/V2 format name consistency for all built-in file formats.
All other built-in file formats already have consistent naming between V1 `FileFormat.toString` and V2 `FileTable.formatName`:
| Format | V1 `toString` | V2 `formatName` | Match |
|---------|---------------|-----------------|-------|
| Parquet | Parquet | Parquet | Yes |
| ORC | ORC | ORC | Yes |
| JSON | JSON | JSON | Yes |
| CSV | CSV | CSV | Yes |
| Text | Text | Text | Yes |
| Avro | Avro | **AVRO** | **No** |
"Avro" is not an acronym (unlike ORC/JSON/CSV); the Apache Avro project uses title case everywhere (https://avro.apache.org/).
### Why are the changes needed?
Follow the existing format precedent for consistency.
### Does this PR introduce _any_ user-facing change?
Yes (minor). The `formatName` for `AvroTable` was originally displayed as `AVRO` and is now shown as `Avro`.
### How was this patch tested?
- New test in `FileTableSuite`: verifies `FileTable.formatName` matches `FileFormat.toString` for all built-in formats (parquet, orc, json, csv, text).
- New test in `AvroV2Suite`: same check for avro.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code
Closes #55319 from LuciferYang/SPARK-56457.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../org/apache/spark/sql/v2/avro/AvroTable.scala | 2 +-
.../scala/org/apache/spark/sql/avro/AvroSuite.scala | 16 ++++++++++++++--
.../sql/execution/datasources/v2/FileTableSuite.scala | 19 +++++++++++++++++++
3 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
index e898253be116..abcea9a2a238 100644
--- a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
+++ b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala
@@ -51,5 +51,5 @@ case class AvroTable(
override def supportsDataType(dataType: DataType): Boolean = AvroUtils.supportsDataType(dataType)
- override def formatName: String = "AVRO"
+ override def formatName: String = "Avro"
}
diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index c55113ec3ef7..fac19a3b8bcf 100644
--- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -22,7 +22,7 @@ import java.net.URI
import java.nio.file.{Files, Paths, StandardCopyOption}
import java.sql.{Date, Timestamp}
import java.time.{LocalDate, LocalDateTime}
-import java.util.UUID
+import java.util.{Collections => JCollections, UUID}
import scala.jdk.CollectionConverters._
@@ -44,7 +44,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA, UTC}
import org.apache.spark.sql.execution.{FormattedMode, SparkPlan}
import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, FilePartition}
-import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
+import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileDataSourceV2, FileTable}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.LegacyBehaviorPolicy
import org.apache.spark.sql.internal.LegacyBehaviorPolicy._
@@ -3483,6 +3483,18 @@ class AvroV2Suite extends AvroSuite with ExplainSuiteHelper {
}
}
+ test("SPARK-56457: Avro V2 formatName matches V1 FileFormat.toString") {
+ val v2Provider = DataSource.lookupDataSourceV2("avro", spark.sessionState.conf)
+ assert(v2Provider.isDefined)
+ val dsV2 = v2Provider.get.asInstanceOf[FileDataSourceV2]
+ val v1Format = dsV2.fallbackFileFormat.getDeclaredConstructor().newInstance()
+ val emptyProps = JCollections.emptyMap[String, String]()
+ val v2Table = dsV2.getTable(
+ new StructType(), Array.empty, emptyProps).asInstanceOf[FileTable]
+ assert(v2Table.formatName == v1Format.toString,
+ s"V2 formatName '${v2Table.formatName}' != V1 toString '${v1Format.toString}'")
+ }
+
test("Geospatial types are not supported in Avro") {
withTempDir { dir =>
// Temporary directory for writing the test data.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala
index 0d18e3bf809e..0269bb464c87 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.datasources.v2
+import java.util.{Collections => JCollections}
+
import scala.jdk.CollectionConverters._
import org.apache.hadoop.fs.FileStatus
@@ -95,6 +97,23 @@ class FileTableSuite extends QueryTest with SharedSparkSession {
}
}
+ test("SPARK-56457: V2 FileTable.formatName matches V1 FileFormat.toString") {
+ withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") {
+ allFileBasedDataSources.foreach { format =>
+ val v2Provider = DataSource.lookupDataSourceV2(format, spark.sessionState.conf)
+ assert(v2Provider.isDefined, s"Expected V2 provider for $format")
+ val dsV2 = v2Provider.get.asInstanceOf[FileDataSourceV2]
+ val v1Format = dsV2.fallbackFileFormat.getDeclaredConstructor().newInstance()
+ val emptyProps = JCollections.emptyMap[String, String]()
+ val v2Table = dsV2.getTable(
+ new StructType(), Array.empty, emptyProps).asInstanceOf[FileTable]
+ assert(v2Table.formatName == v1Format.toString,
+ s"V2 formatName '${v2Table.formatName}' != " +
+ s"V1 toString '${v1Format.toString}' for format '$format'")
+ }
+ }
+ }
+
allFileBasedDataSources.foreach { format =>
test("SPARK-49519, SPARK-50287: Merge options of table and relation when " +
s"constructing ScanBuilder and WriteBuilder in FileFormat - $format") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org