You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2026/02/10 22:31:18 UTC
(hudi) 01/01: fix: Upgrade Spark 3.2.x and 3.3.x and fix parquet schema conversion
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.12.3-spark-upgrade
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c54fd2870dc568337bafe81b191ba3dfce09da63
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Tue Feb 10 14:30:47 2026 -0800
fix: Upgrade Spark 3.2.x and 3.3.x and fix parquet schema conversion
---
README.md | 2 +-
.../datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala | 3 +++
pom.xml | 4 ++--
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index d389754ca216..573d72642ef4 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ mvn clean javadoc:aggregate -Pjavadocs
### Build with different Spark versions
-The default Spark 2.x version supported is 2.4.4. The default Spark 3.x version, corresponding to `spark3` profile is 3.3.1.
+The default Spark 2.x version supported is 2.4.4. The default Spark 3.x version, corresponding to `spark3` profile is 3.3.4.
Refer to the table below for building with different Spark and Scala versions.
| Maven build options | Expected Spark bundle jar name | Notes |
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
index ae686d33a31b..8d4bd46cb6e0 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
@@ -95,6 +95,9 @@ class Spark32PlusHoodieParquetFileFormat(private val shouldAppendPartitionValues
hadoopConf.setBoolean(
SQLConf.PARQUET_INT96_AS_TIMESTAMP.key,
sparkSession.sessionState.conf.isParquetINT96AsTimestamp)
+ hadoopConf.setBoolean(
+ SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key,
+ sparkSession.sessionState.conf.legacyParquetNanosAsLong)
val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA)
// For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself,
diff --git a/pom.xml b/pom.xml
index b6ffc2b307b9..9ba76ec95703 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,8 +148,8 @@
<flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_2.12</flink.hadoop.compatibility.artifactId>
<rocksdbjni.version>5.17.2</rocksdbjni.version>
<spark31.version>3.1.3</spark31.version>
- <spark32.version>3.2.3</spark32.version>
- <spark33.version>3.3.1</spark33.version>
+ <spark32.version>3.2.4</spark32.version>
+ <spark33.version>3.3.4</spark33.version>
<hudi.spark.module>hudi-spark2</hudi.spark.module>
<!-- NOTE: Different Spark versions might require different number of shared
modules being incorporated, hence we're creating multiple placeholders