You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by jo...@apache.org on 2024/08/16 03:55:01 UTC
(hudi) branch master updated: [HUDI-7918] Remove support of Spark 3.0, 3.1, and 3.2 (#11692)
This is an automated email from the ASF dual-hosted git repository.
jonvex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ef8a7ad4f17 [HUDI-7918] Remove support of Spark 3.0, 3.1, and 3.2 (#11692)
ef8a7ad4f17 is described below
commit ef8a7ad4f17a6fe5ad54d1d03a35b32f027b880b
Author: Jon Vexler <jb...@gmail.com>
AuthorDate: Thu Aug 15 23:54:55 2024 -0400
[HUDI-7918] Remove support of Spark 3.0, 3.1, and 3.2 (#11692)
* squash all commits
* fix azure
* only download the docker images once
* fix typo
* remove debug print lines
---------
Co-authored-by: Jonathan Vexler <=>
---
.github/workflows/release_candidate_validation.yml | 32 +-
README.md | 7 +-
azure-pipelines-20230430.yml | 2 -
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 9 -
.../org/apache/hudi/SparkAdapterSupport.scala | 6 -
.../hudi/testutils/HoodieClientTestUtils.java | 2 +-
.../SparkClientFunctionalTestHarness.java | 2 +-
hudi-spark-datasource/README.md | 10 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 68 +-
.../functional/TestHiveTableSchemaEvolution.java | 2 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 2 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 212 +-
.../TestConvertFilterToCatalystExpression.scala | 4 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 74 +-
.../functional/TestSecondaryIndexPruning.scala | 2 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 2 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 10 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 2 +-
.../benchmark/CowTableReadBenchmark.scala | 2 +-
.../ReadAndWriteWithoutAvroBenchmark.scala | 2 +-
.../hudi/command/index/TestFunctionalIndex.scala | 10 +-
.../sql/hudi/command/index/TestIndexSyntax.scala | 2 +-
.../hudi/command/index/TestSecondaryIndex.scala | 6 +-
.../TestNestedSchemaPruningOptimization.scala | 8 +-
.../apache/spark/sql/hudi/ddl/TestAlterTable.scala | 2 +-
.../spark/sql/hudi/ddl/TestCreateTable.scala | 4 +-
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 30 +-
.../hudi/dml/TestHoodieTableValuedFunction.scala | 14 +-
.../spark/sql/hudi/dml/TestInsertTable.scala | 8 +-
.../spark/sql/hudi/dml/TestMergeIntoTable.scala | 18 +-
.../TestMergeIntoTableWithNonRecordKeyField.scala | 2 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 2 +-
.../spark/sql/hudi/dml/TestRepairTable.scala | 2 +-
.../spark/sql/hudi/dml/TestTimeTravelTable.scala | 12 +-
.../apache/hudi/spark3/internal/ReflectUtil.java | 17 +-
...Spark3HoodieVectorizedParquetRecordReader.java} | 4 +-
.../org/apache/hudi/Spark3DefaultSource.scala} | 2 +-
.../apache/spark/sql/ResolveInsertionBase.scala | 1 +
.../spark/sql/adapter/BaseSpark3Adapter.scala | 4 +-
.../HoodieFileSystemViewTableValuedFunction.scala | 0
.../logcal/HoodieMetadataTableValuedFunction.scala | 0
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 0
.../catalyst/plans/logcal/HoodieTableChanges.scala | 0
.../logcal/HoodieTimelineTableValuedFunction.scala | 0
.../spark/sql/catalyst/plans/logical/Index.scala | 0
.../Spark3ParquetSchemaEvolutionUtils.scala | 46 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala} | 8 +-
.../sql/hudi/analysis/TableValuedFunctions.scala | 0
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 15 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 0
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 0
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../sql/hudi/logical/TimeTravelRelation.scala | 0
hudi-spark-datasource/hudi-spark3.0.x/pom.xml | 291 --
...Spark30HoodieVectorizedParquetRecordReader.java | 187 --
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark30DefaultSource.scala | 24 -
.../org/apache/hudi/Spark30HoodieFileScanRDD.scala | 35 -
.../spark/sql/HoodieSpark30CatalogUtils.scala | 29 -
.../sql/HoodieSpark30CatalystExpressionUtils.scala | 157 -
.../spark/sql/HoodieSpark30CatalystPlanUtils.scala | 94 -
.../spark/sql/HoodieSpark30SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_0Adapter.scala | 139 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 493 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 390 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 115 -
.../sql/avro/HoodieSpark3_0AvroDeserializer.scala | 34 -
.../sql/avro/HoodieSpark3_0AvroSerializer.scala | 29 -
.../catalyst/analysis/HoodieSpark30Analysis.scala | 189 --
.../sql/catalyst/plans/logical/HStatments.scala | 66 -
.../HoodieSpark30PartitionedFileUtils.scala | 53 -
.../datasources/Spark30NestedSchemaPruning.scala | 272 --
.../Spark30LegacyHoodieParquetFileFormat.scala | 418 ---
.../datasources/parquet/Spark30ParquetReader.scala | 247 --
.../Spark30ParquetSchemaEvolutionUtils.scala | 58 -
.../hudi/Spark30ResolveHudiAlterTableCommand.scala | 297 --
.../hudi/command/Spark30AlterTableCommand.scala | 320 --
.../sql/parser/HoodieSpark30SqlAstBuilder.scala | 125 -
.../parser/HoodieSpark3_0ExtendedSqlParser.scala | 34 -
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 337 --
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 291 --
...Spark31HoodieVectorizedParquetRecordReader.java | 188 --
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark31DefaultSource.scala | 24 -
.../org/apache/hudi/Spark31HoodieFileScanRDD.scala | 35 -
.../spark/sql/HoodieSpark31CatalogUtils.scala | 30 -
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 114 -
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 94 -
.../spark/sql/HoodieSpark31SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 141 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 497 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 389 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 103 -
.../sql/avro/HoodieSpark3_1AvroDeserializer.scala | 37 -
.../sql/avro/HoodieSpark3_1AvroSerializer.scala | 29 -
.../catalyst/analysis/HoodieSpark31Analysis.scala | 215 --
.../sql/catalyst/plans/logical/HStatments.scala | 66 -
.../HoodieSpark31PartitionedFileUtils.scala | 53 -
.../datasources/Spark31NestedSchemaPruning.scala | 198 --
.../Spark31LegacyHoodieParquetFileFormat.scala | 425 ---
.../datasources/parquet/Spark31ParquetReader.scala | 267 --
.../Spark31ParquetSchemaEvolutionUtils.scala | 61 -
.../hudi/Spark31ResolveHudiAlterTableCommand.scala | 283 --
.../hudi/command/Spark31AlterTableCommand.scala | 320 --
.../sql/parser/HoodieSpark31SqlAstBuilder.scala | 125 -
.../parser/HoodieSpark3_1ExtendedSqlParser.scala | 35 -
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 331 --
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 330 --
.../src/main/antlr4/imports/SqlBase.g4 | 1940 -----------
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 40 -
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark32HoodieFileScanRDD.scala | 32 -
.../spark/sql/HoodieSpark32CatalogUtils.scala | 29 -
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 117 -
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 148 -
.../spark/sql/HoodieSpark32SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 144 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 513 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 450 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 117 -
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 32 -
.../sql/avro/HoodieSpark3_2AvroSerializer.scala | 29 -
.../HoodieSpark32PartitionedFileUtils.scala | 53 -
.../datasources/Spark32NestedSchemaPruning.scala | 199 --
.../parquet/Spark32DataSourceUtils.scala | 77 -
.../Spark32LegacyHoodieParquetFileFormat.scala | 520 ---
.../datasources/parquet/Spark32ParquetReader.scala | 339 --
.../hudi/Spark32ResolveHudiAlterTableCommand.scala | 70 -
.../sql/hudi/analysis/HoodieSpark32Analysis.scala | 66 -
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 3480 --------------------
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 199 --
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 337 --
.../hudi-spark3.2plus-common/pom.xml | 243 --
.../Spark32PlusParquetSchemaEvolutionUtils.scala | 66 -
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 7 -
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 126 +-
.../datasources/parquet/Spark33ParquetReader.scala | 2 +-
hudi-spark-datasource/hudi-spark3.4.x/pom.xml | 7 -
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 126 +-
.../datasources/parquet/Spark34ParquetReader.scala | 2 +-
hudi-spark-datasource/hudi-spark3.5.x/pom.xml | 7 -
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 130 +-
.../datasources/parquet/Spark35ParquetReader.scala | 3 +-
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 2 +
.../utilities/streamer/SchedulerConfGenerator.java | 2 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
.../deltastreamer/TestSchedulerConfGenerator.java | 2 +-
.../functional/TestHoodieSnapshotExporter.java | 2 +-
.../base/build_flink1146hive313spark302.sh | 26 -
.../base/build_flink1146hive313spark313.sh | 26 -
.../base/build_flink1146hive313spark323.sh | 26 -
.../base/build_flink1153hive313spark323.sh | 26 -
packaging/bundle-validation/ci_run.sh | 12 -
packaging/bundle-validation/run_docker_java17.sh | 30 -
packaging/hudi-integ-test-bundle/pom.xml | 3 +-
packaging/hudi-spark-bundle/pom.xml | 3 +-
packaging/hudi-utilities-bundle/pom.xml | 3 +-
pom.xml | 155 +-
scripts/release/deploy_staging_jars.sh | 21 +-
scripts/release/validate_staged_bundles.sh | 4 +-
175 files changed, 401 insertions(+), 20704 deletions(-)
diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml
index b9b668cc80b..5f55005df93 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -25,39 +25,23 @@ jobs:
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.18'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.0'
+ sparkRuntime: 'spark3.5.1'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.18'
- sparkProfile: 'spark3'
- sparkRuntime: 'spark3.5.0'
- - scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.18'
+ flinkProfile: 'flink1.17'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.0'
+ sparkRuntime: 'spark3.5.1'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.18'
+ flinkProfile: 'flink1.16'
sparkProfile: 'spark3.4'
- sparkRuntime: 'spark3.4.0'
+ sparkRuntime: 'spark3.4.3'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.17'
+ flinkProfile: 'flink1.15'
sparkProfile: 'spark3.3'
- sparkRuntime: 'spark3.3.2'
+ sparkRuntime: 'spark3.3.4'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.16'
+ flinkProfile: 'flink1.14'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.1'
- - scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.15'
- sparkProfile: 'spark3.2'
- sparkRuntime: 'spark3.2.3'
- - scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.14'
- sparkProfile: 'spark3.1'
- sparkRuntime: 'spark3.1.3'
- - scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.14'
- sparkProfile: 'spark3.0'
- sparkRuntime: 'spark3.0.2'
- scalaProfile: 'scala-2.11'
flinkProfile: 'flink1.14'
sparkProfile: 'spark'
diff --git a/README.md b/README.md
index 6879f102203..1c819592dea 100644
--- a/README.md
+++ b/README.md
@@ -94,14 +94,11 @@ Refer to the table below for building with different Spark and Scala versions.
| Maven build options | Expected Spark bundle jar name | Notes |
|:--------------------------|:---------------------------------------------|:-------------------------------------------------|
-| (empty) | hudi-spark3.2-bundle_2.12 | For Spark 3.2.x and Scala 2.12 (default options) |
+| (empty) | hudi-spark3.5-bundle_2.12 | For Spark 3.5.x and Scala 2.12 (default options) |
| `-Dspark2.4 -Dscala-2.11` | hudi-spark2.4-bundle_2.11 | For Spark 2.4.4 and Scala 2.11 |
-| `-Dspark3.0` | hudi-spark3.0-bundle_2.12 | For Spark 3.0.x and Scala 2.12 |
-| `-Dspark3.1` | hudi-spark3.1-bundle_2.12 | For Spark 3.1.x and Scala 2.12 |
-| `-Dspark3.2` | hudi-spark3.2-bundle_2.12 | For Spark 3.2.x and Scala 2.12 (same as default) |
| `-Dspark3.3` | hudi-spark3.3-bundle_2.12 | For Spark 3.3.x and Scala 2.12 |
| `-Dspark3.4` | hudi-spark3.4-bundle_2.12 | For Spark 3.4.x and Scala 2.12 |
-| `-Dspark3.5 -Dscala-2.12` | hudi-spark3.5-bundle_2.12 | For Spark 3.5.x and Scala 2.12 |
+| `-Dspark3.5 -Dscala-2.12` | hudi-spark3.5-bundle_2.12 | For Spark 3.5.x and Scala 2.12 (same as default) |
| `-Dspark3.5 -Dscala-2.13` | hudi-spark3.5-bundle_2.13 | For Spark 3.5.x and Scala 2.13 |
| `-Dspark2 -Dscala-2.11` | hudi-spark-bundle_2.11 (legacy bundle name) | For Spark 2.4.4 and Scala 2.11 |
| `-Dspark2 -Dscala-2.12` | hudi-spark-bundle_2.12 (legacy bundle name) | For Spark 2.4.4 and Scala 2.12 |
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 615312ea983..a1a9b5d4e7b 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -62,7 +62,6 @@ parameters:
- 'hudi-spark-datasource'
- 'hudi-spark-datasource/hudi-spark'
- 'hudi-spark-datasource/hudi-spark3.5.x'
- - 'hudi-spark-datasource/hudi-spark3.2plus-common'
- 'hudi-spark-datasource/hudi-spark3-common'
- 'hudi-spark-datasource/hudi-spark-common'
- name: job6UTModules
@@ -89,7 +88,6 @@ parameters:
- '!hudi-spark-datasource'
- '!hudi-spark-datasource/hudi-spark'
- '!hudi-spark-datasource/hudi-spark3.5.x'
- - '!hudi-spark-datasource/hudi-spark3.2plus-common'
- '!hudi-spark-datasource/hudi-spark3-common'
- '!hudi-spark-datasource/hudi-spark-common'
- name: job6FTModules
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
index 82e4f218f65..b8bbe2ed395 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
@@ -48,19 +48,10 @@ private[hudi] trait SparkVersionsSupport {
def isSpark2: Boolean = getSparkVersion.startsWith("2.")
def isSpark3: Boolean = getSparkVersion.startsWith("3.")
- def isSpark3_0: Boolean = getSparkVersion.startsWith("3.0")
- def isSpark3_1: Boolean = getSparkVersion.startsWith("3.1")
- def isSpark3_2: Boolean = getSparkVersion.startsWith("3.2")
def isSpark3_3: Boolean = getSparkVersion.startsWith("3.3")
def isSpark3_4: Boolean = getSparkVersion.startsWith("3.4")
def isSpark3_5: Boolean = getSparkVersion.startsWith("3.5")
- def gteqSpark3_0: Boolean = getSparkVersion >= "3.0"
- def gteqSpark3_1: Boolean = getSparkVersion >= "3.1"
- def gteqSpark3_1_3: Boolean = getSparkVersion >= "3.1.3"
- def gteqSpark3_2: Boolean = getSparkVersion >= "3.2"
- def gteqSpark3_2_1: Boolean = getSparkVersion >= "3.2.1"
- def gteqSpark3_2_2: Boolean = getSparkVersion >= "3.2.2"
def gteqSpark3_3: Boolean = getSparkVersion >= "3.3"
def gteqSpark3_3_2: Boolean = getSparkVersion >= "3.3.2"
def gteqSpark3_4: Boolean = getSparkVersion >= "3.4"
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkAdapterSupport.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkAdapterSupport.scala
index 09229d74b20..9d9a5a10085 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkAdapterSupport.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkAdapterSupport.scala
@@ -39,12 +39,6 @@ object SparkAdapterSupport {
"org.apache.spark.sql.adapter.Spark3_4Adapter"
} else if (HoodieSparkUtils.isSpark3_3) {
"org.apache.spark.sql.adapter.Spark3_3Adapter"
- } else if (HoodieSparkUtils.isSpark3_2) {
- "org.apache.spark.sql.adapter.Spark3_2Adapter"
- } else if (HoodieSparkUtils.isSpark3_1) {
- "org.apache.spark.sql.adapter.Spark3_1Adapter"
- } else if (HoodieSparkUtils.isSpark3_0) {
- "org.apache.spark.sql.adapter.Spark3_0Adapter"
} else {
"org.apache.spark.sql.adapter.Spark2Adapter"
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
index 3eb1da0eae9..1b041002233 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
@@ -104,7 +104,7 @@ public class HoodieClientTestUtils {
sparkConf.set("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
}
- if (canLoadClass("org.apache.spark.sql.hudi.catalog.HoodieCatalog") && HoodieSparkUtils.gteqSpark3_2()) {
+ if (canLoadClass("org.apache.spark.sql.hudi.catalog.HoodieCatalog") && HoodieSparkUtils.gteqSpark3_3()) {
sparkConf.set("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.hudi.catalog.HoodieCatalog");
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
index 79dda856367..7cfaa5c42c0 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
@@ -117,7 +117,7 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
Map<String, String> sqlConf = new HashMap<>();
sqlConf.put("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
- if (HoodieSparkUtils.gteqSpark3_2()) {
+ if (HoodieSparkUtils.gteqSpark3_3()) {
sqlConf.put("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.hudi.catalog.HoodieCatalog");
}
diff --git a/hudi-spark-datasource/README.md b/hudi-spark-datasource/README.md
index dd1796991c8..c5e61f17d50 100644
--- a/hudi-spark-datasource/README.md
+++ b/hudi-spark-datasource/README.md
@@ -21,9 +21,9 @@ This repo contains the code that integrate Hudi with Spark. The repo is split in
`hudi-spark`
`hudi-spark2`
-`hudi-spark3.1.x`
-`hudi-spark3.2.x`
`hudi-spark3.3.x`
+`hudi-spark3.4.x`
+`hudi-spark3.5.x`
`hudi-spark2-common`
`hudi-spark3-common`
`hudi-spark-common`
@@ -31,9 +31,9 @@ This repo contains the code that integrate Hudi with Spark. The repo is split in
* hudi-spark is the module that contains the code that both spark2 & spark3 version would share, also contains the antlr4
file that supports spark sql on spark 2.x version.
* hudi-spark2 is the module that contains the code that compatible with spark 2.x versions.
-* hudi-spark3.1.x is the module that contains the code that compatible with spark3.1.x and spark3.0.x version.
-* hudi-spark3.2.x is the module that contains the code that compatible with spark 3.2.x versions.
-* hudi-spark3.3.x is the module that contains the code that compatible with spark 3.3.x+ versions.
+* hudi-spark3.3.x is the module that contains the code that compatible with spark3.3.x versions.
+* hudi-spark3.4.x is the module that contains the code that compatible with spark 3.4.x versions.
+* hudi-spark3.5.x is the module that contains the code that compatible with spark 3.5.x versions.
* hudi-spark2-common is the module that contains the code that would be reused between spark2.x versions, right now the module
has no class since hudi only supports spark 2.4.4 version, and it acts as the placeholder when packaging hudi-spark-bundle module.
* hudi-spark3-common is the module that contains the code that would be reused between spark3.x versions.
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
index c2327ac2e3b..35059daee86 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
@@ -50,59 +50,38 @@ object HoodieAnalysis extends SparkAdapterSupport {
// For more details please check out the scala-doc of the rule
val adaptIngestionTargetLogicalRelations: RuleBuilder = session => AdaptIngestionTargetLogicalRelations(session)
- if (!HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.isSpark2) {
//Add or correct resolution of MergeInto
// the way we load the class via reflection is diff across spark2 and spark3 and hence had to split it out.
- if (HoodieSparkUtils.isSpark2) {
- val resolveReferencesClass = "org.apache.spark.sql.catalyst.analysis.HoodieSpark2Analysis$ResolveReferences"
- val sparkResolveReferences: RuleBuilder =
- session => ReflectionUtils.loadClass(resolveReferencesClass, session).asInstanceOf[Rule[LogicalPlan]]
- // TODO elaborate on the ordering
- rules += (adaptIngestionTargetLogicalRelations, sparkResolveReferences)
- } else if (HoodieSparkUtils.isSpark3_0) {
- val resolveReferencesClass = "org.apache.spark.sql.catalyst.analysis.HoodieSpark30Analysis$ResolveReferences"
- val sparkResolveReferences: RuleBuilder = {
- session => instantiateKlass(resolveReferencesClass, session)
- }
- // TODO elaborate on the ordering
- rules += (adaptIngestionTargetLogicalRelations, sparkResolveReferences)
- } else if (HoodieSparkUtils.isSpark3_1) {
- val resolveReferencesClass = "org.apache.spark.sql.catalyst.analysis.HoodieSpark31Analysis$ResolveReferences"
- val sparkResolveReferences: RuleBuilder =
- session => instantiateKlass(resolveReferencesClass, session)
- // TODO elaborate on the ordering
- rules += (adaptIngestionTargetLogicalRelations, sparkResolveReferences)
- } else {
- throw new IllegalStateException("Impossible to be here")
- }
+ val resolveReferencesClass = "org.apache.spark.sql.catalyst.analysis.HoodieSpark2Analysis$ResolveReferences"
+ val sparkResolveReferences: RuleBuilder =
+ session => ReflectionUtils.loadClass(resolveReferencesClass, session).asInstanceOf[Rule[LogicalPlan]]
+ // TODO elaborate on the ordering
+ rules += (adaptIngestionTargetLogicalRelations, sparkResolveReferences)
} else {
rules += adaptIngestionTargetLogicalRelations
val dataSourceV2ToV1FallbackClass = if (HoodieSparkUtils.isSpark3_5)
"org.apache.spark.sql.hudi.analysis.HoodieSpark35DataSourceV2ToV1Fallback"
else if (HoodieSparkUtils.isSpark3_4)
"org.apache.spark.sql.hudi.analysis.HoodieSpark34DataSourceV2ToV1Fallback"
- else if (HoodieSparkUtils.isSpark3_3)
- "org.apache.spark.sql.hudi.analysis.HoodieSpark33DataSourceV2ToV1Fallback"
else {
- // Spark 3.2.x
- "org.apache.spark.sql.hudi.analysis.HoodieSpark32DataSourceV2ToV1Fallback"
+ // Spark 3.3.x
+ "org.apache.spark.sql.hudi.analysis.HoodieSpark33DataSourceV2ToV1Fallback"
}
val dataSourceV2ToV1Fallback: RuleBuilder =
session => instantiateKlass(dataSourceV2ToV1FallbackClass, session)
- val spark32PlusResolveReferencesClass = "org.apache.spark.sql.hudi.analysis.HoodieSpark32PlusResolveReferences"
- val spark32PlusResolveReferences: RuleBuilder =
- session => instantiateKlass(spark32PlusResolveReferencesClass, session)
+ val spark3ResolveReferencesClass = "org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences"
+ val spark3ResolveReferences: RuleBuilder =
+ session => instantiateKlass(spark3ResolveReferencesClass, session)
// NOTE: PLEASE READ CAREFULLY BEFORE CHANGING
//
// It's critical for this rules to follow in this order; re-ordering this rules might lead to changes in
// behavior of Spark's analysis phase (for ex, DataSource V2 to V1 fallback might not kick in before other rules,
// leading to all relations resolving as V2 instead of current expectation of them being resolved as V1)
- rules ++= Seq(dataSourceV2ToV1Fallback, spark32PlusResolveReferences)
- }
+ rules ++= Seq(dataSourceV2ToV1Fallback, spark3ResolveReferences)
- if (HoodieSparkUtils.isSpark3) {
val resolveAlterTableCommandsClass =
if (HoodieSparkUtils.gteqSpark3_5) {
"org.apache.spark.sql.hudi.Spark35ResolveHudiAlterTableCommand"
@@ -110,12 +89,6 @@ object HoodieAnalysis extends SparkAdapterSupport {
"org.apache.spark.sql.hudi.Spark34ResolveHudiAlterTableCommand"
} else if (HoodieSparkUtils.gteqSpark3_3) {
"org.apache.spark.sql.hudi.Spark33ResolveHudiAlterTableCommand"
- } else if (HoodieSparkUtils.gteqSpark3_2) {
- "org.apache.spark.sql.hudi.Spark32ResolveHudiAlterTableCommand"
- } else if (HoodieSparkUtils.gteqSpark3_1) {
- "org.apache.spark.sql.hudi.Spark31ResolveHudiAlterTableCommand"
- } else if (HoodieSparkUtils.gteqSpark3_0) {
- "org.apache.spark.sql.hudi.Spark30ResolveHudiAlterTableCommand"
} else {
throw new IllegalStateException("Unsupported Spark version")
}
@@ -142,8 +115,8 @@ object HoodieAnalysis extends SparkAdapterSupport {
session => HoodiePostAnalysisRule(session)
)
- if (HoodieSparkUtils.gteqSpark3_2) {
- val spark3PostHocResolutionClass = "org.apache.spark.sql.hudi.analysis.HoodieSpark32PlusPostAnalysisRule"
+ if (HoodieSparkUtils.isSpark3) {
+ val spark3PostHocResolutionClass = "org.apache.spark.sql.hudi.analysis.HoodieSpark3PostAnalysisRule"
val spark3PostHocResolution: RuleBuilder =
session => instantiateKlass(spark3PostHocResolutionClass, session)
@@ -158,22 +131,15 @@ object HoodieAnalysis extends SparkAdapterSupport {
// Default rules
)
- if (HoodieSparkUtils.gteqSpark3_0) {
+ if (HoodieSparkUtils.isSpark3) {
val nestedSchemaPruningClass =
if (HoodieSparkUtils.gteqSpark3_5) {
"org.apache.spark.sql.execution.datasources.Spark35NestedSchemaPruning"
} else if (HoodieSparkUtils.gteqSpark3_4) {
"org.apache.spark.sql.execution.datasources.Spark34NestedSchemaPruning"
- } else if (HoodieSparkUtils.gteqSpark3_3) {
- "org.apache.spark.sql.execution.datasources.Spark33NestedSchemaPruning"
- } else if (HoodieSparkUtils.gteqSpark3_2) {
- "org.apache.spark.sql.execution.datasources.Spark32NestedSchemaPruning"
- } else if (HoodieSparkUtils.gteqSpark3_1) {
- // spark 3.1
- "org.apache.spark.sql.execution.datasources.Spark31NestedSchemaPruning"
} else {
- // spark 3.0
- "org.apache.spark.sql.execution.datasources.Spark30NestedSchemaPruning"
+ // spark 3.3
+ "org.apache.spark.sql.execution.datasources.Spark33NestedSchemaPruning"
}
val nestedSchemaPruningRule = ReflectionUtils.loadClass(nestedSchemaPruningClass).asInstanceOf[Rule[LogicalPlan]]
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHiveTableSchemaEvolution.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHiveTableSchemaEvolution.java
index 806f7754423..49988fa1687 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHiveTableSchemaEvolution.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHiveTableSchemaEvolution.java
@@ -92,7 +92,7 @@ public class TestHiveTableSchemaEvolution {
@ParameterizedTest
@ValueSource(strings = {"cow", "mor"})
public void testHiveReadSchemaEvolutionTable(String tableType) throws Exception {
- if (HoodieSparkUtils.gteqSpark3_1()) {
+ if (HoodieSparkUtils.gteqSpark3_3()) {
String tableName = "hudi_test" + new Date().getTime();
String path = new Path(basePath.toAbsolutePath().toString()).toUri().toString();
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestOrcBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestOrcBootstrap.java
index 99ae55a9ee1..f8c8527f413 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestOrcBootstrap.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestOrcBootstrap.java
@@ -191,7 +191,7 @@ public class TestOrcBootstrap extends HoodieSparkClientTestBase {
private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode, BootstrapMode modeForRegexMatch) throws Exception {
// NOTE: Hudi doesn't support Orc in Spark < 3.0
// Please check HUDI-4496 for more details
- if (!HoodieSparkUtils.gteqSpark3_0()) {
+ if (!HoodieSparkUtils.gteqSpark3_3()) {
return;
}
String keyGeneratorClass = partitioned ? SimpleKeyGenerator.class.getCanonicalName()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
index 86c5ba48ef2..7cc76bea198 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
@@ -301,58 +301,52 @@ class TestAvroSchemaResolutionSupport extends HoodieClientTestBase with ScalaAss
@MethodSource(Array("testArgs"))
def testArrayOfStructsAddNewColumn(isCow: Boolean, useFileGroupReader: Boolean): Unit = {
// test to add a field to a STRUCT in a column of ARRAY< STRUCT<..> > type
-
- // there is a bug on Spark3 that will prevent Array[Map/Struct] schema evolved tables form being read
- // bug fix: https://github.com/apache/spark/commit/32a393395ee43b573ae75afba591b587ca51879b
- // bug fix is only available Spark >= v3.1.3
- if (HoodieSparkUtils.isSpark2 || (HoodieSparkUtils.isSpark3 && HoodieSparkUtils.gteqSpark3_1_3)) {
- val tempRecordPath = basePath + "/record_tbl/"
- val arrayStructData = Seq(
- Row(1, 100, List(Row("Java", "XX", 120), Row("Scala", "XA", 300)), "aaa")
- )
- val arrayStructSchema = new StructType()
- .add("id", IntegerType)
- .add("userid", IntegerType)
- .add("language", ArrayType(new StructType()
- .add("name", StringType)
- .add("author", StringType)
- .add("pages", IntegerType)))
+ val tempRecordPath = basePath + "/record_tbl/"
+ val arrayStructData = Seq(
+ Row(1, 100, List(Row("Java", "XX", 120), Row("Scala", "XA", 300)), "aaa")
+ )
+ val arrayStructSchema = new StructType()
+ .add("id", IntegerType)
+ .add("userid", IntegerType)
+ .add("language", ArrayType(new StructType()
.add("name", StringType)
- val df1 = spark.createDataFrame(spark.sparkContext.parallelize(arrayStructData), arrayStructSchema)
- df1.printSchema()
- df1.show(false)
-
- // recreate table
- initialiseTable(df1, tempRecordPath, isCow)
-
- // add a column to array of struct
- val newArrayStructData = Seq(
- Row(2, 200, List(Row("JavaV2", "XXX", 130, 20), Row("ScalaV2", "XXA", 310, 40)), "bbb")
- )
- val newArrayStructSchema = new StructType()
- .add("id", IntegerType)
- .add("userid", IntegerType)
- .add("language", ArrayType(new StructType()
- .add("name", StringType)
- .add("author", StringType)
- .add("pages", IntegerType)
- .add("progress", IntegerType)
- ))
+ .add("author", StringType)
+ .add("pages", IntegerType)))
+ .add("name", StringType)
+ val df1 = spark.createDataFrame(spark.sparkContext.parallelize(arrayStructData), arrayStructSchema)
+ df1.printSchema()
+ df1.show(false)
+
+ // recreate table
+ initialiseTable(df1, tempRecordPath, isCow)
+
+ // add a column to array of struct
+ val newArrayStructData = Seq(
+ Row(2, 200, List(Row("JavaV2", "XXX", 130, 20), Row("ScalaV2", "XXA", 310, 40)), "bbb")
+ )
+ val newArrayStructSchema = new StructType()
+ .add("id", IntegerType)
+ .add("userid", IntegerType)
+ .add("language", ArrayType(new StructType()
.add("name", StringType)
- val df2 = spark.createDataFrame(spark.sparkContext.parallelize(newArrayStructData), newArrayStructSchema)
- df2.printSchema()
- df2.show(false)
- // upsert
- upsertData(df2, tempRecordPath, isCow)
+ .add("author", StringType)
+ .add("pages", IntegerType)
+ .add("progress", IntegerType)
+ ))
+ .add("name", StringType)
+ val df2 = spark.createDataFrame(spark.sparkContext.parallelize(newArrayStructData), newArrayStructSchema)
+ df2.printSchema()
+ df2.show(false)
+ // upsert
+ upsertData(df2, tempRecordPath, isCow)
- // read out the table
- val readDf = spark.read.format("hudi")
- .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), useFileGroupReader)
- .load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
- }
+ // read out the table
+ val readDf = spark.read.format("hudi")
+ .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), useFileGroupReader)
+ .load(tempRecordPath)
+ readDf.printSchema()
+ readDf.show(false)
+ readDf.foreach(_ => {})
}
@ParameterizedTest
@@ -575,71 +569,65 @@ class TestAvroSchemaResolutionSupport extends HoodieClientTestBase with ScalaAss
@MethodSource(Array("testArgs"))
def testArrayOfMapsStructAddField(isCow: Boolean, useFileGroupReader: Boolean): Unit = {
// test to add a field to a STRUCT in a column of ARRAY< MAP< k,STRUCT<..> > > type
+ val tempRecordPath = basePath + "/record_tbl/"
+ val arrayMapData = Seq(
+ Row(1, 100,
+ List(
+ Map("2022-12-01" -> Row("a1", "b1", 20)),
+ Map("2022-12-02" -> Row("a2", "b2", 30))
+ ),
+ "aaa")
+ )
+ val innerStructSchema = new StructType()
+ .add("col1", StringType)
+ .add("col2", StringType)
+ .add("col3", IntegerType)
+ val arrayMapSchema = new StructType()
+ .add("id", IntegerType)
+ .add("userid", IntegerType)
+ .add("structcol", ArrayType(
+ new MapType(StringType, innerStructSchema, true)))
+ .add("name", StringType)
+ val df1 = spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData), arrayMapSchema)
+ df1.printSchema()
+ df1.show(false)
- // there is a bug on Spark3 that will prevent Array[Map/Struct] schema evolved tables form being read
- // bug fix: https://github.com/apache/spark/commit/32a393395ee43b573ae75afba591b587ca51879b
- // bug fix is only available Spark >= v3.1.3
- if (HoodieSparkUtils.isSpark2 || (HoodieSparkUtils.isSpark3 && HoodieSparkUtils.gteqSpark3_1_3)) {
- val tempRecordPath = basePath + "/record_tbl/"
- val arrayMapData = Seq(
- Row(1, 100,
- List(
- Map("2022-12-01" -> Row("a1", "b1", 20)),
- Map("2022-12-02" -> Row("a2", "b2", 30))
- ),
- "aaa")
- )
- val innerStructSchema = new StructType()
- .add("col1", StringType)
- .add("col2", StringType)
- .add("col3", IntegerType)
- val arrayMapSchema = new StructType()
- .add("id", IntegerType)
- .add("userid", IntegerType)
- .add("structcol", ArrayType(
- new MapType(StringType, innerStructSchema, true)))
- .add("name", StringType)
- val df1 = spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData), arrayMapSchema)
- df1.printSchema()
- df1.show(false)
-
- // recreate table
- initialiseTable(df1, tempRecordPath, isCow)
-
- // add a new column
- val newArrayMapData = Seq(
- Row(2, 200,
- List(
- Map("2022-12-01" -> Row("a3", "b3", 20, 40)),
- Map("2022-12-02" -> Row("a4", "b4", 30, 40))
- ),
- "bbb")
- )
- val newInnerStructSchema = new StructType()
- .add("col1", StringType)
- .add("col2", StringType)
- .add("col3", IntegerType)
- .add("col4", IntegerType)
- val newArrayMapSchema = new StructType()
- .add("id", IntegerType)
- .add("userid", IntegerType)
- .add("structcol", ArrayType(
- new MapType(StringType, newInnerStructSchema, true)))
- .add("name", StringType)
- val df2 = spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData), newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
- // upsert
- upsertData(df2, tempRecordPath, isCow)
+ // recreate table
+ initialiseTable(df1, tempRecordPath, isCow)
- // read out the table
- val readDf = spark.read.format("hudi")
- .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), useFileGroupReader)
- .load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
- }
+ // add a new column
+ val newArrayMapData = Seq(
+ Row(2, 200,
+ List(
+ Map("2022-12-01" -> Row("a3", "b3", 20, 40)),
+ Map("2022-12-02" -> Row("a4", "b4", 30, 40))
+ ),
+ "bbb")
+ )
+ val newInnerStructSchema = new StructType()
+ .add("col1", StringType)
+ .add("col2", StringType)
+ .add("col3", IntegerType)
+ .add("col4", IntegerType)
+ val newArrayMapSchema = new StructType()
+ .add("id", IntegerType)
+ .add("userid", IntegerType)
+ .add("structcol", ArrayType(
+ new MapType(StringType, newInnerStructSchema, true)))
+ .add("name", StringType)
+ val df2 = spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData), newArrayMapSchema)
+ df2.printSchema()
+ df2.show(false)
+ // upsert
+ upsertData(df2, tempRecordPath, isCow)
+
+ // read out the table
+ val readDf = spark.read.format("hudi")
+ .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), useFileGroupReader)
+ .load(tempRecordPath)
+ readDf.printSchema()
+ readDf.show(false)
+ readDf.foreach(_ => {})
}
@ParameterizedTest
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestConvertFilterToCatalystExpression.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestConvertFilterToCatalystExpression.scala
index e199dc055f5..8bbfa818e21 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestConvertFilterToCatalystExpression.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestConvertFilterToCatalystExpression.scala
@@ -80,7 +80,7 @@ class TestConvertFilterToCatalystExpression {
private def checkConvertFilter(filter: Filter, expectExpression: String): Unit = {
// [SPARK-25769][SPARK-34636][SPARK-34626][SQL] sql method in UnresolvedAttribute,
// AttributeReference and Alias don't quote qualified names properly
- val removeQuotesIfNeed = if (expectExpression != null && HoodieSparkUtils.gteqSpark3_2) {
+ val removeQuotesIfNeed = if (expectExpression != null && HoodieSparkUtils.gteqSpark3_3) {
expectExpression.replace("`", "")
} else {
expectExpression
@@ -97,7 +97,7 @@ class TestConvertFilterToCatalystExpression {
private def checkConvertFilters(filters: Array[Filter], expectExpression: String): Unit = {
// [SPARK-25769][SPARK-34636][SPARK-34626][SQL] sql method in UnresolvedAttribute,
// AttributeReference and Alias don't quote qualified names properly
- val removeQuotesIfNeed = if (expectExpression != null && HoodieSparkUtils.gteqSpark3_2) {
+ val removeQuotesIfNeed = if (expectExpression != null && HoodieSparkUtils.gteqSpark3_3) {
expectExpression.replace("`", "")
} else {
expectExpression
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index b4d8da7b131..3e2fe0963ac 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -1274,7 +1274,7 @@ object TestHoodieSparkSqlWriter {
// NOTE: Hudi doesn't support Orc in Spark < 3.0
// Please check HUDI-4496 for more details
- val targetScenarios = if (HoodieSparkUtils.gteqSpark3_0) {
+ val targetScenarios = if (HoodieSparkUtils.gteqSpark3_3) {
parquetScenarios ++ orcScenarios
} else {
parquetScenarios
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
index 1783c286c59..39092887a82 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
@@ -35,7 +35,7 @@ import scala.collection.JavaConverters
class TestHoodieSparkUtils {
@ParameterizedTest
- @ValueSource(strings = Array("2.4.4", "3.1.0", "3.2.0", "3.3.0"))
+ @ValueSource(strings = Array("2.4.4", "3.3.0", "3.3.2", "3.4.0", "3.5.0"))
def testSparkVersionCheckers(sparkVersion: String): Unit = {
val vsMock = new SparkVersionsSupport {
override def getSparkVersion: String = sparkVersion
@@ -46,45 +46,61 @@ class TestHoodieSparkUtils {
assertTrue(vsMock.isSpark2)
assertFalse(vsMock.isSpark3)
- assertFalse(vsMock.isSpark3_1)
- assertFalse(vsMock.isSpark3_0)
- assertFalse(vsMock.isSpark3_2)
- assertFalse(vsMock.gteqSpark3_1)
- assertFalse(vsMock.gteqSpark3_1_3)
- assertFalse(vsMock.gteqSpark3_2)
-
- case "3.1.0" =>
+ assertFalse(vsMock.isSpark3_3)
+ assertFalse(vsMock.isSpark3_4)
+ assertFalse(vsMock.isSpark3_5)
+ assertFalse(vsMock.gteqSpark3_3)
+ assertFalse(vsMock.gteqSpark3_3_2)
+ assertFalse(vsMock.gteqSpark3_4)
+ assertFalse(vsMock.gteqSpark3_5)
+
+ case "3.3.0" =>
assertTrue(vsMock.isSpark3)
- assertTrue(vsMock.isSpark3_1)
- assertTrue(vsMock.gteqSpark3_1)
+ assertTrue(vsMock.isSpark3_3)
+ assertTrue(vsMock.gteqSpark3_3)
assertFalse(vsMock.isSpark2)
- assertFalse(vsMock.isSpark3_0)
- assertFalse(vsMock.isSpark3_2)
- assertFalse(vsMock.gteqSpark3_1_3)
- assertFalse(vsMock.gteqSpark3_2)
+ assertFalse(vsMock.isSpark3_4)
+ assertFalse(vsMock.isSpark3_5)
+ assertFalse(vsMock.gteqSpark3_3_2)
+ assertFalse(vsMock.gteqSpark3_4)
+ assertFalse(vsMock.gteqSpark3_5)
- case "3.2.0" =>
+ case "3.3.2" =>
assertTrue(vsMock.isSpark3)
- assertTrue(vsMock.isSpark3_2)
- assertTrue(vsMock.gteqSpark3_1)
- assertTrue(vsMock.gteqSpark3_1_3)
- assertTrue(vsMock.gteqSpark3_2)
+ assertTrue(vsMock.isSpark3_3)
+ assertTrue(vsMock.gteqSpark3_3)
+ assertTrue(vsMock.gteqSpark3_3_2)
+
assertFalse(vsMock.isSpark2)
- assertFalse(vsMock.isSpark3_0)
- assertFalse(vsMock.isSpark3_1)
+ assertFalse(vsMock.isSpark3_4)
+ assertFalse(vsMock.isSpark3_5)
+ assertFalse(vsMock.gteqSpark3_4)
+ assertFalse(vsMock.gteqSpark3_5)
- case "3.3.0" =>
+ case "3.4.0" =>
+ assertTrue(vsMock.isSpark3)
+ assertTrue(vsMock.isSpark3_4)
+ assertTrue(vsMock.gteqSpark3_3)
+ assertTrue(vsMock.gteqSpark3_3_2)
+ assertTrue(vsMock.gteqSpark3_4)
+
+ assertFalse(vsMock.isSpark2)
+ assertFalse(vsMock.isSpark3_3)
+ assertFalse(vsMock.isSpark3_5)
+
+ case "3.5.0" =>
assertTrue(vsMock.isSpark3)
- assertTrue(vsMock.gteqSpark3_1)
- assertTrue(vsMock.gteqSpark3_1_3)
- assertTrue(vsMock.gteqSpark3_2)
+ assertTrue(vsMock.isSpark3_5)
+ assertTrue(vsMock.gteqSpark3_3)
+ assertTrue(vsMock.gteqSpark3_3_2)
+ assertTrue(vsMock.gteqSpark3_4)
+ assertTrue(vsMock.gteqSpark3_5)
- assertFalse(vsMock.isSpark3_2)
assertFalse(vsMock.isSpark2)
- assertFalse(vsMock.isSpark3_0)
- assertFalse(vsMock.isSpark3_1)
+ assertFalse(vsMock.isSpark3_3)
+ assertFalse(vsMock.isSpark3_4)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
index e90e752502e..8d6448b0f82 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
@@ -36,7 +36,7 @@ class TestSecondaryIndexPruning extends SecondaryIndexTestBase {
@Test
def testSecondaryIndexWithFilters(): Unit = {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
var hudiOpts = commonOpts
hudiOpts = hudiOpts + (
DataSourceWriteOptions.TABLE_TYPE.key -> HoodieTableType.COPY_ON_WRITE.name(),
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
index bf5cb404393..66b1390d2a1 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
@@ -144,7 +144,7 @@ class TestSparkSqlCoreFlow extends HoodieSparkSqlTestBase {
assertEquals(commitInstantTime3, countsPerCommit(0).get(0).toString)
- val timeTravelDf = if (HoodieSparkUtils.gteqSpark3_2_1) {
+ val timeTravelDf = if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(s"select * from $tableName timestamp as of '$commitInstantTime2'").cache()
} else {
//HUDI-5265
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
index 12b4c32bbaa..c0fa7bccd5e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
@@ -236,10 +236,10 @@ class TestSparkSqlWithCustomKeyGenerator extends HoodieSparkSqlTestBase {
}
test("Test table property isolation for partition path field config "
- + "with custom key generator for Spark 3.1 and above") {
- // Only testing Spark 3.1 and above as lower Spark versions do not support
+ + "with custom key generator for Spark 3.3 and above") {
+ // Only testing Spark 3.3 and above as lower Spark versions do not support
// ALTER TABLE .. SET TBLPROPERTIES .. to store table-level properties in Hudi Catalog
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp => {
val tableNameNonPartitioned = generateTableName
val tableNameSimpleKey = generateTableName
@@ -353,9 +353,9 @@ class TestSparkSqlWithCustomKeyGenerator extends HoodieSparkSqlTestBase {
| """.stripMargin)
}
- // Only testing Spark 3.1 and above as lower Spark versions do not support
+ // Only testing Spark 3.3 and above as lower Spark versions do not support
// ALTER TABLE .. SET TBLPROPERTIES .. to store table-level properties in Hudi Catalog
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
// Now fix the partition path field write config for tableName
spark.sql(
s"""ALTER TABLE $tableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala
index 74837c0007f..45d50a6dcc8 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala
@@ -47,7 +47,7 @@ object BoundInMemoryExecutorBenchmark extends HoodieBenchmarkBase {
def sparkConf(): SparkConf = {
val sparkConf = new SparkConf()
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
sparkConf.set("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala
index 2d9525c934e..3d5b50d3c9b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala
@@ -50,7 +50,7 @@ object CowTableReadBenchmark extends HoodieBenchmarkBase {
def sparkConf(): SparkConf = {
val sparkConf = new SparkConf()
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
sparkConf.set("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala
index 4d02d3ed482..d7f7cf6a9f3 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala
@@ -54,7 +54,7 @@ object ReadAndWriteWithoutAvroBenchmark extends HoodieBenchmarkBase {
def sparkConf(): SparkConf = {
val sparkConf = new SparkConf()
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
sparkConf.set("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala
index c7036b1d86d..98e4f0bd2c0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala
@@ -49,7 +49,7 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
// There is a big difference between Java class loader architecture of versions 1.8 and 17.
// Hive 2.3.7 is compiled with Java 1.8, and the class loader used there throws error when Hive APIs are run on Java 17.
// So we special case this test only for Java 8.
- if (HoodieSparkUtils.gteqSpark3_2 && HoodieTestUtils.getJavaVersion == 8) {
+ if (HoodieSparkUtils.gteqSpark3_3 && HoodieTestUtils.getJavaVersion == 8) {
withTempDir { tmp =>
Seq("mor").foreach { tableType =>
val databaseName = "testdb"
@@ -120,7 +120,7 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
}
test("Test Create Functional Index Syntax") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val databaseName = "default"
@@ -174,7 +174,7 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
}
test("Test Create Functional Index") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val databaseName = "default"
@@ -249,7 +249,7 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
}
test("Test functional index update after initialization") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir(tmp => {
val tableName = generateTableName
val basePath = s"${tmp.getCanonicalPath}/$tableName"
@@ -311,7 +311,7 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
}
test("Test Create Functional Index With Data Skipping") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow").foreach { tableType =>
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala
index 87146b5c0e2..4c047423d65 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
class TestIndexSyntax extends HoodieSparkSqlTestBase {
test("Test Create/Drop/Show/Refresh Index") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val databaseName = "default"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
index da7a3ad348f..d0d5b67a142 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
class TestSecondaryIndex extends HoodieSparkSqlTestBase {
test("Test Create/Show/Drop Secondary Index") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
@@ -96,7 +96,7 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase {
}
test("Test Secondary Index Creation With hudi_metadata TVF") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir {
tmp => {
val tableName = generateTableName
@@ -132,7 +132,7 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase {
}
test("Test Secondary Index Creation Failure For Multiple Fields") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir {
tmp => {
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala
index d9307b8213a..d464e07d748 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala
@@ -42,9 +42,9 @@ class TestNestedSchemaPruningOptimization extends HoodieSparkSqlTestBase with Sp
test("Test NestedSchemaPruning optimization successful") {
withTempDir { tmp =>
- // NOTE: This tests are only relevant for Spark >= 3.1
+ // NOTE: This tests are only relevant for Spark >= 3.3
// TODO extract tests into a separate spark-version-specific module
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${tmp.getCanonicalPath}/$tableName"
@@ -107,9 +107,9 @@ class TestNestedSchemaPruningOptimization extends HoodieSparkSqlTestBase with Sp
test("Test NestedSchemaPruning optimization unsuccessful") {
withTempDir { tmp =>
- // NOTE: This tests are only relevant for Spark >= 3.1
+ // NOTE: This tests are only relevant for Spark >= 3.3
// TODO extract tests into a separate spark-version-specific module
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
// TODO add cow
Seq("mor").foreach { tableType =>
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
index 1a9a028ded1..1e1fd1c1a0f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
@@ -204,7 +204,7 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
Seq(1, "a1", 10.0, 1000, null)
)
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withSQLConf("hoodie.schema.on.read.enable" -> "true") {
spark.sql(s"alter table $tableName2 add columns(hh string comment 'hour time')")
Seq(1, "a1", 10.0, 1000, null, null)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index c5c3d98856c..28c633ee95b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -404,7 +404,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
}
test("Test create table like") {
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
// 1. Test create table from an existing HUDI table
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
@@ -1110,7 +1110,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
)
}
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
test("Test create table with comment") {
val tableName = generateTableName
spark.sql(
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
index 511916f7bb7..385ba06ba5e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
@@ -76,7 +76,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set " + DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION.key + "=upsert")
spark.sql("set hoodie.schema.on.read.enable=true")
// NOTE: This is required since as this tests use type coercions which were only permitted in Spark 2.x
@@ -150,7 +150,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
// NOTE: This is required since as this tests use type coercions which were only permitted in Spark 2.x
// and are disallowed now by default in Spark 3.x
@@ -187,7 +187,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
withTempDir { tmp =>
val tableName = generateTableName
val tablePath = s"${tmp.getCanonicalPath}/$tableName"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
// Create table
spark.sql(
@@ -244,7 +244,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
spark.sql("set hoodie.compact.inline=false")
spark.sql("set hoodie.compact.schedule.inline=false")
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
spark.sql("set " + DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION.key + "=upsert")
// NOTE: This is required since as this tests use type coercions which were only permitted in Spark 2.x
@@ -349,7 +349,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
spark.sql("set " + DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION.key + "=upsert")
spark.sql(
@@ -402,7 +402,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
spark.sql(
s"""
@@ -483,7 +483,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
spark.sql(
s"""
@@ -512,7 +512,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql("set hoodie.schema.on.read.enable=true")
spark.sql(
s"""
@@ -552,7 +552,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
"spark.sql.parquet.enableNestedColumnVectorizedReader" -> "false") {
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(
s"""
|create table $tableName (
@@ -639,7 +639,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("COPY_ON_WRITE", "MERGE_ON_READ").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
val dataGen = new HoodieTestDataGenerator
val schema = HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA
@@ -713,7 +713,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
// for complex schema.
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
val dataGen = new QuickstartUtils.DataGenerator
val inserts = QuickstartUtils.convertToStringList(dataGen.generateInserts(10)).asScala.toSeq
val df = spark.read.json(spark.sparkContext.parallelize(inserts, 2))
@@ -813,7 +813,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq("COPY_ON_WRITE", "MERGE_ON_READ").foreach { tableType =>
val tableName = generateTableName
val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
// adding a struct column to force reads to use non-vectorized readers
spark.sql(
s"""
@@ -861,7 +861,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
withTempDir { tmp =>
// Using INMEMORY index for mor table so that log files will be created instead of parquet
val tableName = generateTableName
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(
s"""
|create table $tableName (
@@ -901,7 +901,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
withTempDir { tmp =>
// Using INMEMORY index for mor table so that log files will be created instead of parquet
val tableName = generateTableName
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(
s"""
|create table $tableName (
@@ -969,7 +969,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
withTempDir { tmp =>
// Using INMEMORY index for mor table so that log files will be created instead of parquet
val tableName = generateTableName
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(
s"""
|create table $tableName (
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala
index 2409d783de3..e41d05138d5 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
test(s"Test hudi_query Table-Valued Function") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
@@ -88,7 +88,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_table_changes latest_state") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq(
("cow", true),
@@ -195,7 +195,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_filesystem_view") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq(
("cow", true),
@@ -258,7 +258,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_table_changes cdc") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq(
("cow", true),
@@ -415,7 +415,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_query_timeline") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq(
("cow", true),
@@ -563,7 +563,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_metadata Table-Valued Function") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
@@ -629,7 +629,7 @@ class TestHoodieTableValuedFunction extends HoodieSparkSqlTestBase {
}
test(s"Test hudi_metadata Table-Valued Function For PARTITION_STATS index") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
index d5a7ea3567c..9390df7b73a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
@@ -1462,7 +1462,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
withTempDir { tmp =>
val tableName = s"spark_catalog.default.$generateTableName"
// Create a partitioned table
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
spark.sql(
s"""
|create table $tableName (
@@ -2590,7 +2590,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
}
test(s"Test INSERT INTO with upsert operation type") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
Seq("mor").foreach { tableType =>
val tableName = generateTableName
@@ -2831,7 +2831,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
test("Test insert into with special cols") {
withTempDir { tmp =>
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
val targetTableA = generateTableName
val tablePathA = s"${tmp.getCanonicalPath}/$targetTableA"
if (HoodieSparkUtils.isSpark3_4) {
@@ -2909,7 +2909,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
test("Test insert overwrite with special cols") {
withTempDir { tmp =>
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
val targetTableA = generateTableName
val tablePathA = s"${tmp.getCanonicalPath}/$targetTableA"
if (HoodieSparkUtils.isSpark3_4) {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala
index cfb0475b36b..9da5a1a6818 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala
@@ -128,9 +128,9 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
* Refer to: `org.apache.spark.sql.catalyst.parser.AstBuilder#visitMergeIntoTable`
*
*/
- test("Test MergeInto with more than once update actions for spark >= 3.1.x") {
+ test("Test MergeInto with more than once update actions for spark >= 3.3.x") {
- if (HoodieSparkUtils.gteqSpark3_1) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
val targetTable = generateTableName
spark.sql(
@@ -361,10 +361,8 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
)
// Delete with condition expression.
- val errorMessage = if (HoodieSparkUtils.gteqSpark3_2) {
+ val errorMessage = if (HoodieSparkUtils.gteqSpark3_3) {
"Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.id = (s0.s_id + 1)`"
- } else if (HoodieSparkUtils.gteqSpark3_1) {
- "Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.`id` = (s0.`s_id` + 1)`"
} else {
"Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.`id` = (s0.`s_id` + 1)`;"
}
@@ -575,7 +573,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
//
// 2) set source column name to be different with target column
//
- val errorMessage = if (HoodieSparkUtils.gteqSpark3_1) {
+ val errorMessage = if (HoodieSparkUtils.gteqSpark3_3) {
"Failed to resolve pre-combine field `v` w/in the source-table output"
} else {
"Failed to resolve pre-combine field `v` w/in the source-table output;"
@@ -640,10 +638,8 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
// Delete data with a condition expression on primaryKey field
// 1) set source column name to be same as target column
//
- val complexConditionsErrorMessage = if (HoodieSparkUtils.gteqSpark3_2) {
+ val complexConditionsErrorMessage = if (HoodieSparkUtils.gteqSpark3_3) {
"Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.id = (s0.id + 1)`"
- } else if (HoodieSparkUtils.gteqSpark3_1) {
- "Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.`id` = (s0.`id` + 1)`"
} else {
"Only simple conditions of the form `t.id = s.id` are allowed on the primary-key and partition path column. Found `t0.`id` = (s0.`id` + 1)`;"
}
@@ -675,7 +671,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
//
// 2.a) set source column name to be different with target column (should fail unable to match pre-combine field)
//
- val failedToResolveErrorMessage = if (HoodieSparkUtils.gteqSpark3_1) {
+ val failedToResolveErrorMessage = if (HoodieSparkUtils.gteqSpark3_3) {
"Failed to resolve pre-combine field `v` w/in the source-table output"
} else {
"Failed to resolve pre-combine field `v` w/in the source-table output;"
@@ -1142,7 +1138,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
// Can't down-cast incoming dataset's primary-key w/o loss of precision (should fail)
- val errorMsg = if (HoodieSparkUtils.gteqSpark3_2) {
+ val errorMsg = if (HoodieSparkUtils.gteqSpark3_3) {
"Invalid MERGE INTO matching condition: s0.id: can't cast s0.id (of LongType) to IntegerType"
} else {
"Invalid MERGE INTO matching condition: s0.`id`: can't cast s0.`id` (of LongType) to IntegerType"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala
index 2a9bad788c7..a1b6d53819f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala
@@ -133,7 +133,7 @@ class TestMergeIntoTableWithNonRecordKeyField extends HoodieSparkSqlTestBase wit
|""".stripMargin)
if (sparkSqlOptimizedWrites) {
- val errorMessage2 = if (HoodieSparkUtils.gteqSpark3_1) {
+ val errorMessage2 = if (HoodieSparkUtils.gteqSpark3_3) {
"Hudi tables with primary key are required to match on all primary key colums. Column: 'name' not found"
} else {
"Hudi tables with primary key are required to match on all primary key colums. Column: 'name' not found;"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
index 818d4867290..2da26ebb052 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
@@ -344,7 +344,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| preCombineField = '_ts'
|)""".stripMargin)
- val failedToResolveErrorMessage = if (HoodieSparkUtils.gteqSpark3_1) {
+ val failedToResolveErrorMessage = if (HoodieSparkUtils.gteqSpark3_3) {
"Failed to resolve pre-combine field `_ts` w/in the source-table output"
} else {
"Failed to resolve pre-combine field `_ts` w/in the source-table output;"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala
index e4191157e6c..b6f96a1d785 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala
@@ -134,7 +134,7 @@ class TestRepairTable extends HoodieSparkSqlTestBase {
}
test("Test msck repair partitioned table [add/drop/sync] partitions") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
Seq("true", "false").foreach { hiveStylePartitionEnable =>
withTempDir { tmp =>
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
index 183480fe691..ab5657d8d3a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
class TestTimeTravelTable extends HoodieSparkSqlTestBase {
test("Test Insert and Update Record with time travel") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withRecordType()(withTempDir { tmp =>
val tableName1 = generateTableName
spark.sql(
@@ -67,7 +67,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
test("Test Insert Into Records with time travel To new Table") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
// Create Non-Partitioned table
val tableName1 = generateTableName
@@ -139,7 +139,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
test("Test Two Table's Union Join with time travel") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withRecordType()(withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
@@ -225,7 +225,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
test("Test Unsupported syntax can be parsed") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
checkAnswer("select 1 distribute by 1")(Seq(1))
withTempDir { dir =>
val path = dir.toURI.getPath
@@ -237,7 +237,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
test("Test Select Record with time travel and Repartition") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withTempDir { tmp =>
val tableName = generateTableName
spark.sql(
@@ -280,7 +280,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
test("Test Time Travel With Schema Evolution") {
- if (HoodieSparkUtils.gteqSpark3_2) {
+ if (HoodieSparkUtils.gteqSpark3_3) {
withRecordType()(withTempDir { tmp =>
spark.sql("set hoodie.schema.on.read.enable=true")
val tableName = generateTableName
diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
index c726777876f..9164210b76c 100644
--- a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
@@ -17,8 +17,6 @@
package org.apache.hudi.spark3.internal;
-import org.apache.hudi.HoodieSparkUtils;
-
import org.apache.spark.sql.catalyst.util.DateFormatter;
import java.lang.reflect.Method;
@@ -29,17 +27,10 @@ public class ReflectUtil {
public static DateFormatter getDateFormatter(ZoneId zoneId) {
try {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
- if (HoodieSparkUtils.gteqSpark3_2()) {
- Class<?> clazz = loader.loadClass(DateFormatter.class.getName());
- Method applyMethod = clazz.getDeclaredMethod("apply");
- applyMethod.setAccessible(true);
- return (DateFormatter)applyMethod.invoke(null);
- } else {
- Class<?> clazz = loader.loadClass(DateFormatter.class.getName());
- Method applyMethod = clazz.getDeclaredMethod("apply", ZoneId.class);
- applyMethod.setAccessible(true);
- return (DateFormatter)applyMethod.invoke(null, zoneId);
- }
+ Class<?> clazz = loader.loadClass(DateFormatter.class.getName());
+ Method applyMethod = clazz.getDeclaredMethod("apply");
+ applyMethod.setAccessible(true);
+ return (DateFormatter)applyMethod.invoke(null);
} catch (Exception e) {
throw new RuntimeException("Error in apply DateFormatter", e);
}
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark3HoodieVectorizedParquetRecordReader.java
similarity index 97%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java
rename to hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark3HoodieVectorizedParquetRecordReader.java
index d42fe746b3a..f1d10d28855 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark3HoodieVectorizedParquetRecordReader.java
@@ -36,7 +36,7 @@ import java.time.ZoneId;
import java.util.HashMap;
import java.util.Map;
-public class Spark32PlusHoodieVectorizedParquetRecordReader extends VectorizedParquetRecordReader {
+public class Spark3HoodieVectorizedParquetRecordReader extends VectorizedParquetRecordReader {
// save the col type change info.
private Map<Integer, Pair<DataType, DataType>> typeChangeInfos;
@@ -63,7 +63,7 @@ public class Spark32PlusHoodieVectorizedParquetRecordReader extends VectorizedPa
private int batchIdx = 0;
private int numBatched = 0;
- public Spark32PlusHoodieVectorizedParquetRecordReader(
+ public Spark3HoodieVectorizedParquetRecordReader(
ZoneId convertTz,
String datetimeRebaseMode,
String datetimeRebaseTz,
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/hudi/Spark32PlusDefaultSource.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
similarity index 94%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/hudi/Spark32PlusDefaultSource.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
index 339c6a33083..3bc3446d1f1 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/hudi/Spark32PlusDefaultSource.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.sources.DataSourceRegister
* there are no regressions in performance
* Please check out HUDI-4178 for more details
*/
-class Spark32PlusDefaultSource extends DefaultSource with DataSourceRegister /* with TableProvider */ {
+class Spark3DefaultSource extends DefaultSource with DataSourceRegister /* with TableProvider */ {
override def shortName(): String = "hudi"
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
similarity index 99%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
index 868e225476d..674b62887ec 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
@@ -19,6 +19,7 @@
package org.apache.spark.sql
import org.apache.hudi.SparkAdapterSupport
+
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast}
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, Project}
diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
index 919eda52c8d..539ce713c53 100644
--- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
@@ -47,9 +47,7 @@ import scala.collection.JavaConverters._
* Base implementation of [[SparkAdapter]] for Spark 3.x branch
*/
abstract class BaseSpark3Adapter extends SparkAdapter with Logging {
-
- // JsonUtils for Support Spark Version >= 3.2
- if (HoodieSparkUtils.gteqSpark3_2) JsonUtils.registerModules()
+ JsonUtils.registerModules()
private val cache = new ConcurrentHashMap[ZoneId, DateFormatter](1)
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieFileSystemViewTableValuedFunction.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieFileSystemViewTableValuedFunction.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieFileSystemViewTableValuedFunction.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieFileSystemViewTableValuedFunction.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieMetadataTableValuedFunction.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieMetadataTableValuedFunction.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieMetadataTableValuedFunction.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieMetadataTableValuedFunction.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieQuery.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieQuery.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieQuery.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieQuery.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTableChanges.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTableChanges.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTableChanges.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTableChanges.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTimelineTableValuedFunction.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTimelineTableValuedFunction.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTimelineTableValuedFunction.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTimelineTableValuedFunction.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Index.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Index.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Index.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Index.scala
diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3ParquetSchemaEvolutionUtils.scala
similarity index 89%
rename from hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3ParquetSchemaEvolutionUtils.scala
index 4996a668af5..a42beaabd75 100644
--- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3ParquetSchemaEvolutionUtils.scala
@@ -15,11 +15,8 @@
* limitations under the License.
*/
+package org.apache.spark.sql.execution.datasources.parquet
-package org.apache.spark.sql.execution.datasources
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.client.utils.SparkInternalSchemaConverter
import org.apache.hudi.common.fs.FSUtils
@@ -30,18 +27,22 @@ import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.parquet.hadoop.metadata.FileMetaData
import org.apache.spark.sql.HoodieSchemaUtils
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, UnsafeProjection}
import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
-import org.apache.spark.sql.execution.datasources.Spark3ParquetSchemaEvolutionUtils.pruneInternalSchema
-import org.apache.spark.sql.execution.datasources.parquet.{HoodieParquetFileFormatHelper, ParquetReadSupport}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, UnsafeProjection}
+import org.apache.spark.sql.execution.datasources.parquet.Spark3ParquetSchemaEvolutionUtils.pruneInternalSchema
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType}
+import java.time.ZoneId
+
import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
-abstract class Spark3ParquetSchemaEvolutionUtils(sharedConf: Configuration,
+class Spark3ParquetSchemaEvolutionUtils(sharedConf: Configuration,
filePath: Path,
requiredSchema: StructType,
partitionSchema: StructType,
@@ -178,6 +179,35 @@ abstract class Spark3ParquetSchemaEvolutionUtils(sharedConf: Configuration,
GenerateUnsafeProjection.generate(castSchema, newFullSchema)
}
}
+
+ def buildVectorizedReader(convertTz: ZoneId,
+ datetimeRebaseMode: String,
+ datetimeRebaseTz: String,
+ int96RebaseMode: String,
+ int96RebaseTz: String,
+ useOffHeap: Boolean,
+ capacity: Int): VectorizedParquetRecordReader = {
+ if (shouldUseInternalSchema) {
+ new Spark3HoodieVectorizedParquetRecordReader(
+ convertTz,
+ datetimeRebaseMode,
+ datetimeRebaseTz,
+ int96RebaseMode,
+ int96RebaseTz,
+ useOffHeap,
+ capacity,
+ typeChangeInfos)
+ } else {
+ new VectorizedParquetRecordReader(
+ convertTz,
+ datetimeRebaseMode,
+ datetimeRebaseTz,
+ int96RebaseMode,
+ int96RebaseTz,
+ useOffHeap,
+ capacity)
+ }
+ }
}
object Spark3ParquetSchemaEvolutionUtils {
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
similarity index 97%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
index 1c422b473de..d917adb7d94 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelpe
import org.apache.spark.sql.execution.datasources.{DataSource, LogicalRelation}
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.isMetaField
import org.apache.spark.sql.hudi.ProvidesHoodieConfig
-import org.apache.spark.sql.hudi.analysis.HoodieSpark32PlusAnalysis.{HoodieV1OrV2Table, ResolvesToHudiTable}
+import org.apache.spark.sql.hudi.analysis.HoodieSpark3Analysis.{HoodieV1OrV2Table, ResolvesToHudiTable}
import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table
import org.apache.spark.sql.hudi.command.{AlterHoodieTableDropPartitionCommand, ShowHoodieTablePartitionsCommand, TruncateHoodieTableCommand}
@@ -52,7 +52,7 @@ import org.apache.spark.sql.hudi.command.{AlterHoodieTableDropPartitionCommand,
/**
* Rule for resolve hoodie's extended syntax or rewrite some logical plan.
*/
-case class HoodieSpark32PlusResolveReferences(spark: SparkSession) extends Rule[LogicalPlan]
+case class HoodieSpark3ResolveReferences(spark: SparkSession) extends Rule[LogicalPlan]
with SparkAdapterSupport with ProvidesHoodieConfig {
def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
@@ -310,7 +310,7 @@ case class HoodieSpark32PlusResolveReferences(spark: SparkSession) extends Rule[
* Rule replacing resolved Spark's commands (not working for Hudi tables out-of-the-box) with
* corresponding Hudi implementations
*/
-case class HoodieSpark32PlusPostAnalysisRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
+case class HoodieSpark3PostAnalysisRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan match {
case ShowPartitions(MatchResolvedTable(_, id, HoodieV1OrV2Table(_)), specOpt, _) =>
@@ -338,7 +338,7 @@ case class HoodieSpark32PlusPostAnalysisRule(sparkSession: SparkSession) extends
}
}
-object HoodieSpark32PlusAnalysis extends SparkAdapterSupport {
+object HoodieSpark3Analysis extends SparkAdapterSupport {
private[sql] object HoodieV1OrV2Table {
def unapply(table: Table): Option[CatalogTable] = table match {
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/TableValuedFunctions.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/TableValuedFunctions.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/TableValuedFunctions.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/TableValuedFunctions.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
similarity index 97%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
index 7e196e00c6e..bdde33b5845 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
@@ -7,13 +7,14 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.spark.sql.hudi.catalog
@@ -40,7 +41,7 @@ import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, ColumnChan
import org.apache.spark.sql.connector.catalog._
import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
import org.apache.spark.sql.execution.datasources.DataSource
-import org.apache.spark.sql.hudi.analysis.HoodieSpark32PlusAnalysis.HoodieV1OrV2Table
+import org.apache.spark.sql.hudi.analysis.HoodieSpark3Analysis.HoodieV1OrV2Table
import org.apache.spark.sql.hudi.catalog.HoodieCatalog.{buildPartitionTransforms, isTablePartitioned}
import org.apache.spark.sql.hudi.command._
import org.apache.spark.sql.hudi.{HoodieSqlCommonUtils, ProvidesHoodieConfig}
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala
similarity index 100%
rename from hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala
rename to hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/pom.xml b/hudi-spark-datasource/hudi-spark3.0.x/pom.xml
deleted file mode 100644
index 4878274304f..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/pom.xml
+++ /dev/null
@@ -1,291 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hudi-spark-datasource</artifactId>
- <groupId>org.apache.hudi</groupId>
- <version>1.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>hudi-spark3.0.x_2.12</artifactId>
- <version>1.0.0-SNAPSHOT</version>
-
- <name>hudi-spark3.0.x_2.12</name>
- <packaging>jar</packaging>
-
- <properties>
- <main.basedir>${project.parent.parent.basedir}</main.basedir>
- </properties>
-
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>${scala-maven-plugin.version}</version>
- <configuration>
- <args>
- <arg>-nobootcp</arg>
- <arg>-target:jvm-1.8</arg>
- </args>
- <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- </plugins>
- </pluginManagement>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/lib</outputDirectory>
- <overWriteReleases>true</overWriteReleases>
- <overWriteSnapshots>true</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- </executions>
- <configuration>
- <skip>false</skip>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>${skip.hudi-spark3.unit.tests}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark30.version}</version>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark30.version}</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-client</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark3-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Hoodie - Test -->
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-tests-common</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-client-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-client</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-hadoop-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Parquet -->
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-avro</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieVectorizedParquetRecordReader.java b/hudi-spark-datasource/hudi-spark3.0.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieVectorizedParquetRecordReader.java
deleted file mode 100644
index e1607373c1e..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieVectorizedParquetRecordReader.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hudi.client.utils.SparkInternalSchemaConverter;
-import org.apache.hudi.common.util.collection.Pair;
-import org.apache.spark.memory.MemoryMode;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.execution.vectorized.OffHeapColumnVector;
-import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
-import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.StructType;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-import java.io.IOException;
-import java.time.ZoneId;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Spark30HoodieVectorizedParquetRecordReader extends VectorizedParquetRecordReader {
-
- // save the col type change info.
- private Map<Integer, Pair<DataType, DataType>> typeChangeInfos;
-
- private ColumnarBatch columnarBatch;
-
- private Map<Integer, WritableColumnVector> idToColumnVectors;
-
- private WritableColumnVector[] columnVectors;
-
- // The capacity of vectorized batch.
- private int capacity;
-
- // If true, this class returns batches instead of rows.
- private boolean returnColumnarBatch;
-
- // The memory mode of the columnarBatch.
- private final MemoryMode memoryMode;
-
- /**
- * Batch of rows that we assemble and the current index we've returned. Every time this
- * batch is used up (batchIdx == numBatched), we populated the batch.
- */
- private int batchIdx = 0;
- private int numBatched = 0;
-
- public Spark30HoodieVectorizedParquetRecordReader(
- ZoneId convertTz,
- String datetimeRebaseMode,
- boolean useOffHeap,
- int capacity,
- Map<Integer, Pair<DataType, DataType>> typeChangeInfos) {
- super(convertTz, datetimeRebaseMode, useOffHeap, capacity);
- memoryMode = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
- this.typeChangeInfos = typeChangeInfos;
- this.capacity = capacity;
- }
-
- @Override
- public void initBatch(StructType partitionColumns, InternalRow partitionValues) {
- super.initBatch(partitionColumns, partitionValues);
- if (columnVectors == null) {
- columnVectors = new WritableColumnVector[sparkSchema.length() + partitionColumns.length()];
- }
- if (idToColumnVectors == null) {
- idToColumnVectors = new HashMap<>();
- typeChangeInfos.entrySet()
- .stream()
- .forEach(f -> {
- WritableColumnVector vector =
- memoryMode == MemoryMode.OFF_HEAP ? new OffHeapColumnVector(capacity, f.getValue().getLeft()) : new OnHeapColumnVector(capacity, f.getValue().getLeft());
- idToColumnVectors.put(f.getKey(), vector);
- });
- }
- }
-
- @Override
- public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException, UnsupportedOperationException {
- super.initialize(inputSplit, taskAttemptContext);
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- for (Map.Entry<Integer, WritableColumnVector> e : idToColumnVectors.entrySet()) {
- e.getValue().close();
- }
- idToColumnVectors = null;
- columnarBatch = null;
- columnVectors = null;
- }
-
- @Override
- public ColumnarBatch resultBatch() {
- ColumnarBatch currentColumnBatch = super.resultBatch();
- boolean changed = false;
- for (Map.Entry<Integer, Pair<DataType, DataType>> entry : typeChangeInfos.entrySet()) {
- boolean rewrite = SparkInternalSchemaConverter
- .convertColumnVectorType((WritableColumnVector) currentColumnBatch.column(entry.getKey()),
- idToColumnVectors.get(entry.getKey()), currentColumnBatch.numRows());
- if (rewrite) {
- changed = true;
- columnVectors[entry.getKey()] = idToColumnVectors.get(entry.getKey());
- }
- }
- if (changed) {
- if (columnarBatch == null) {
- // fill other vector
- for (int i = 0; i < columnVectors.length; i++) {
- if (columnVectors[i] == null) {
- columnVectors[i] = (WritableColumnVector) currentColumnBatch.column(i);
- }
- }
- columnarBatch = new ColumnarBatch(columnVectors);
- }
- columnarBatch.setNumRows(currentColumnBatch.numRows());
- return columnarBatch;
- } else {
- return currentColumnBatch;
- }
- }
-
- @Override
- public boolean nextBatch() throws IOException {
- boolean result = super.nextBatch();
- if (idToColumnVectors != null) {
- idToColumnVectors.entrySet().stream().forEach(e -> e.getValue().reset());
- }
- numBatched = resultBatch().numRows();
- batchIdx = 0;
- return result;
- }
-
- @Override
- public void enableReturningBatches() {
- returnColumnarBatch = true;
- super.enableReturningBatches();
- }
-
- @Override
- public Object getCurrentValue() {
- if (typeChangeInfos == null || typeChangeInfos.isEmpty()) {
- return super.getCurrentValue();
- }
-
- if (returnColumnarBatch) {
- return columnarBatch == null ? super.getCurrentValue() : columnarBatch;
- }
-
- return columnarBatch == null ? super.getCurrentValue() : columnarBatch.getRow(batchIdx - 1);
- }
-
- @Override
- public boolean nextKeyValue() throws IOException {
- resultBatch();
-
- if (returnColumnarBatch) {
- return nextBatch();
- }
-
- if (batchIdx >= numBatched) {
- if (!nextBatch()) {
- return false;
- }
- }
- ++batchIdx;
- return true;
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/hudi-spark-datasource/hudi-spark3.0.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
deleted file mode 100644
index 4043b10b3b7..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ /dev/null
@@ -1,19 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-org.apache.hudi.Spark30DefaultSource
\ No newline at end of file
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30DefaultSource.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30DefaultSource.scala
deleted file mode 100644
index d1a2766e487..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30DefaultSource.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi
-
-import org.apache.spark.sql.sources.DataSourceRegister
-
-class Spark30DefaultSource extends DefaultSource with DataSourceRegister {
- override def shortName(): String = "hudi"
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30HoodieFileScanRDD.scala
deleted file mode 100644
index c9700541e64..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30HoodieFileScanRDD.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi
-
-import org.apache.hudi.HoodieUnsafeRDD
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile}
-import org.apache.spark.sql.types.StructType
-
-class Spark30HoodieFileScanRDD(@transient private val sparkSession: SparkSession,
- read: PartitionedFile => Iterator[InternalRow],
- @transient filePartitions: Seq[FilePartition])
- extends FileScanRDD(sparkSession, read, filePartitions)
- with HoodieUnsafeRDD {
-
- override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect()
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalogUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalogUtils.scala
deleted file mode 100644
index f8863f52297..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalogUtils.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.connector.expressions.{BucketTransform, NamedReference, Transform}
-
-object HoodieSpark30CatalogUtils extends HoodieSpark3CatalogUtils {
-
- override def unapplyBucketTransform(t: Transform): Option[(Int, Seq[NamedReference], Seq[NamedReference])] =
- t match {
- case BucketTransform(numBuckets, ref) => Some(numBuckets, Seq(ref), Seq.empty)
- case _ => None
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystExpressionUtils.scala
deleted file mode 100644
index c4708be813b..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystExpressionUtils.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering
-import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
-import org.apache.spark.sql.catalyst.expressions.{AnsiCast, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, Exp, Expm1, Expression, FromUTCTimestamp, FromUnixTime, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUTCTimestamp, ToUnixTimestamp, Upper}
-import org.apache.spark.sql.types.{DataType, StructType}
-
-object HoodieSpark30CatalystExpressionUtils extends HoodieSpark3CatalystExpressionUtils {
-
- override def getEncoder(schema: StructType): ExpressionEncoder[Row] = {
- RowEncoder.apply(schema).resolveAndBind()
- }
-
- override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] =
- expr match {
- case Cast(child, dataType, timeZoneId) => Some((child, dataType, timeZoneId))
- case _ => None
- }
-
- override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] = {
- exprs.map {
- _.transform {
- case a: AttributeReference =>
- a.withName(attributes.find(_.semanticEquals(a)).getOrElse(a).name)
- }
- }
- }
-
- // NOTE: This method has been borrowed from Spark 3.1
- override def extractPredicatesWithinOutputSet(condition: Expression,
- outputSet: AttributeSet): Option[Expression] = condition match {
- case org.apache.spark.sql.catalyst.expressions.And(left, right) =>
- val leftResultOptional = extractPredicatesWithinOutputSet(left, outputSet)
- val rightResultOptional = extractPredicatesWithinOutputSet(right, outputSet)
- (leftResultOptional, rightResultOptional) match {
- case (Some(leftResult), Some(rightResult)) => Some(org.apache.spark.sql.catalyst.expressions.And(leftResult, rightResult))
- case (Some(leftResult), None) => Some(leftResult)
- case (None, Some(rightResult)) => Some(rightResult)
- case _ => None
- }
-
- // The Or predicate is convertible when both of its children can be pushed down.
- // That is to say, if one/both of the children can be partially pushed down, the Or
- // predicate can be partially pushed down as well.
- //
- // Here is an example used to explain the reason.
- // Let's say we have
- // condition: (a1 AND a2) OR (b1 AND b2),
- // outputSet: AttributeSet(a1, b1)
- // a1 and b1 is convertible, while a2 and b2 is not.
- // The predicate can be converted as
- // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2)
- // As per the logical in And predicate, we can push down (a1 OR b1).
- case org.apache.spark.sql.catalyst.expressions.Or(left, right) =>
- for {
- lhs <- extractPredicatesWithinOutputSet(left, outputSet)
- rhs <- extractPredicatesWithinOutputSet(right, outputSet)
- } yield org.apache.spark.sql.catalyst.expressions.Or(lhs, rhs)
-
- // Here we assume all the `Not` operators is already below all the `And` and `Or` operators
- // after the optimization rule `BooleanSimplification`, so that we don't need to handle the
- // `Not` operators here.
- case other =>
- if (other.references.subsetOf(outputSet)) {
- Some(other)
- } else {
- None
- }
- }
-
- override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = {
- expr match {
- case OrderPreservingTransformation(attrRef) => Some(attrRef)
- case _ => None
- }
- }
-
- def canUpCast(fromType: DataType, toType: DataType): Boolean =
- Cast.canUpCast(fromType, toType)
-
- override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] =
- expr match {
- case Cast(castedExpr, dataType, timeZoneId) => Some((castedExpr, dataType, timeZoneId, false))
- case AnsiCast(castedExpr, dataType, timeZoneId) => Some((castedExpr, dataType, timeZoneId, true))
- case _ => None
- }
-
- private object OrderPreservingTransformation {
- def unapply(expr: Expression): Option[AttributeReference] = {
- expr match {
- // Date/Time Expressions
- case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case ParseToDate(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
-
- // String Expressions
- case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
-
- // Math Expressions
- // Binary
- case org.apache.spark.sql.catalyst.expressions.Add(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case org.apache.spark.sql.catalyst.expressions.Add(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Multiply(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case Multiply(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Divide(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- // Unary
- case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
-
- // Other
- case cast @ Cast(OrderPreservingTransformation(attrRef), _, _)
- if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef)
-
- // Identity transformation
- case attrRef: AttributeReference => Some(attrRef)
- // No match
- case _ => None
- }
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystPlanUtils.scala
deleted file mode 100644
index 1ea04b512e6..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystPlanUtils.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.ResolvedTable
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema}
-import org.apache.spark.sql.catalyst.planning.ScanOperation
-import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, MergeIntoTable}
-import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
-import org.apache.spark.sql.execution.command.AlterTableRecoverPartitionsCommand
-import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat}
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.types.StructType
-
-object HoodieSpark30CatalystPlanUtils extends HoodieSpark3CatalystPlanUtils {
-
- def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] =
- plan match {
- case ResolvedTable(catalog, identifier, table) => Some((catalog, identifier, table))
- case _ => None
- }
-
- override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = {
- plan match {
- case MergeIntoTable(targetTable, sourceTable, mergeCondition, _, _) =>
- Some((targetTable, sourceTable, mergeCondition))
- case _ => None
- }
- }
-
- override def maybeApplyForNewFileFormat(plan: LogicalPlan): LogicalPlan = {
- plan match {
- case s@ScanOperation(_, _,
- l@LogicalRelation(fs: HadoopFsRelation, _, _, _))
- if fs.fileFormat.isInstanceOf[ParquetFileFormat with HoodieFormatTrait]
- && !fs.fileFormat.asInstanceOf[ParquetFileFormat with HoodieFormatTrait].isProjected =>
- FileFormatUtilsForFileGroupReader.applyNewFileFormatChanges(s, l, fs)
- case _ => plan
- }
- }
-
- override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = ProjectionOverSchema(schema)
-
- override def isRepairTable(plan: LogicalPlan): Boolean = {
- plan.isInstanceOf[AlterTableRecoverPartitionsCommand]
- }
-
- override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = {
- plan match {
- // For Spark >= 3.2.x, AlterTableRecoverPartitionsCommand was renamed RepairTableCommand, and added two new
- // parameters: enableAddPartitions and enableDropPartitions. By setting them to true and false, can restore
- // AlterTableRecoverPartitionsCommand's behavior
- case c: AlterTableRecoverPartitionsCommand =>
- Some((c.tableName, true, false, c.cmd))
- }
- }
-
- /**
- * Commands of managing indexes are not supported for Spark3.0
- */
- override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = None
-
- override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = None
-
- override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = None
-
- override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = None
-
- override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = {
- plan match {
- case insert: InsertIntoStatement =>
- Some((insert.table, Seq.empty, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists))
- case _ =>
- None
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
deleted file mode 100644
index f66fd837c7e..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.SchemaUtils
-
-/**
- * Utils on schema for Spark 3.0.
- */
-object HoodieSpark30SchemaUtils extends HoodieSchemaUtils {
- override def checkColumnNameDuplication(columnNames: Seq[String],
- colType: String,
- caseSensitiveAnalysis: Boolean): Unit = {
- SchemaUtils.checkColumnNameDuplication(columnNames, colType, caseSensitiveAnalysis)
- }
-
- override def toAttributes(struct: StructType): Seq[Attribute] = {
- struct.toAttributes
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_0Adapter.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_0Adapter.scala
deleted file mode 100644
index 8fbcf5a060b..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_0Adapter.scala
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.adapter
-
-import org.apache.avro.Schema
-import org.apache.hadoop.conf.Configuration
-import org.apache.hudi.Spark30HoodieFileScanRDD
-import org.apache.spark.sql._
-import org.apache.spark.sql.avro.{HoodieAvroDeserializer, HoodieAvroSerializer, HoodieSpark3_0AvroDeserializer, HoodieSpark3_0AvroSerializer}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.catalog.CatalogTable
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, Spark30LegacyHoodieParquetFileFormat, Spark30ParquetReader, SparkParquetReader}
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, HoodieSpark30PartitionedFileUtils, HoodieSparkPartitionedFileUtils, PartitionedFile}
-import org.apache.spark.sql.hudi.SparkAdapter
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.parser.{HoodieExtendedParserInterface, HoodieSpark3_0ExtendedSqlParser}
-import org.apache.spark.sql.types.{DataType, Metadata, MetadataBuilder, StructType}
-import org.apache.spark.sql.vectorized.ColumnarUtils
-import org.apache.spark.storage.StorageLevel
-import org.apache.spark.storage.StorageLevel._
-
-/**
- * Implementation of [[SparkAdapter]] for Spark 3.0.x
- */
-class Spark3_0Adapter extends BaseSpark3Adapter {
-
- override def resolveHoodieTable(plan: LogicalPlan): Option[CatalogTable] = {
- super.resolveHoodieTable(plan).orElse {
- EliminateSubqueryAliases(plan) match {
- // First, we need to weed out unresolved plans
- case plan if !plan.resolved => None
- // NOTE: When resolving Hudi table we allow [[Filter]]s and [[Project]]s be applied
- // on top of it
- case PhysicalOperation(_, _, DataSourceV2Relation(table: CatalogTable, _, _, _, _)) if isHoodieTable(table) =>
- Some(table)
- case _ => None
- }
- }
- }
-
- override def isColumnarBatchRow(r: InternalRow): Boolean = ColumnarUtils.isColumnarBatchRow(r)
-
- def createCatalystMetadataForMetaField: Metadata =
- // NOTE: Since [[METADATA_COL_ATTR_KEY]] flag is not available in Spark 2.x,
- // we simply produce an empty [[Metadata]] instance
- new MetadataBuilder().build()
-
- override def getCatalogUtils: HoodieSpark3CatalogUtils = HoodieSpark30CatalogUtils
-
- override def getCatalystPlanUtils: HoodieCatalystPlansUtils = HoodieSpark30CatalystPlanUtils
-
- override def getCatalystExpressionUtils: HoodieCatalystExpressionUtils = HoodieSpark30CatalystExpressionUtils
-
- override def getSchemaUtils: HoodieSchemaUtils = HoodieSpark30SchemaUtils
-
- override def getSparkPartitionedFileUtils: HoodieSparkPartitionedFileUtils = HoodieSpark30PartitionedFileUtils
-
- override def createAvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean): HoodieAvroSerializer =
- new HoodieSpark3_0AvroSerializer(rootCatalystType, rootAvroType, nullable)
-
- override def createAvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType): HoodieAvroDeserializer =
- new HoodieSpark3_0AvroDeserializer(rootAvroType, rootCatalystType)
-
- override def createExtendedSparkParser(spark: SparkSession, delegate: ParserInterface): HoodieExtendedParserInterface =
- new HoodieSpark3_0ExtendedSqlParser(spark, delegate)
-
- override def createLegacyHoodieParquetFileFormat(appendPartitionValues: Boolean): Option[ParquetFileFormat] = {
- Some(new Spark30LegacyHoodieParquetFileFormat(appendPartitionValues))
- }
-
- override def createHoodieFileScanRDD(sparkSession: SparkSession,
- readFunction: PartitionedFile => Iterator[InternalRow],
- filePartitions: Seq[FilePartition],
- readDataSchema: StructType,
- metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = {
- new Spark30HoodieFileScanRDD(sparkSession, readFunction, filePartitions)
- }
-
- override def extractDeleteCondition(deleteFromTable: Command): Expression = {
- deleteFromTable.asInstanceOf[DeleteFromTable].condition.getOrElse(null)
- }
-
- /**
- * Converts instance of [[StorageLevel]] to a corresponding string
- */
- override def convertStorageLevelToString(level: StorageLevel): String = level match {
- case NONE => "NONE"
- case DISK_ONLY => "DISK_ONLY"
- case DISK_ONLY_2 => "DISK_ONLY_2"
- case MEMORY_ONLY => "MEMORY_ONLY"
- case MEMORY_ONLY_2 => "MEMORY_ONLY_2"
- case MEMORY_ONLY_SER => "MEMORY_ONLY_SER"
- case MEMORY_ONLY_SER_2 => "MEMORY_ONLY_SER_2"
- case MEMORY_AND_DISK => "MEMORY_AND_DISK"
- case MEMORY_AND_DISK_2 => "MEMORY_AND_DISK_2"
- case MEMORY_AND_DISK_SER => "MEMORY_AND_DISK_SER"
- case MEMORY_AND_DISK_SER_2 => "MEMORY_AND_DISK_SER_2"
- case OFF_HEAP => "OFF_HEAP"
- case _ => throw new IllegalArgumentException(s"Invalid StorageLevel: $level")
- }
-
- /**
- * Get parquet file reader
- *
- * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc
- * @param sqlConf the [[SQLConf]] used for the read
- * @param options passed as a param to the file format
- * @param hadoopConf some configs will be set for the hadoopConf
- * @return parquet file reader
- */
- override def createParquetFileReader(vectorized: Boolean,
- sqlConf: SQLConf,
- options: Map[String, String],
- hadoopConf: Configuration): SparkParquetReader = {
- Spark30ParquetReader.build(vectorized, sqlConf, options, hadoopConf)
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
deleted file mode 100644
index 67c5e169a49..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ /dev/null
@@ -1,493 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Conversions.DecimalConversion
-import org.apache.avro.LogicalTypes.{TimestampMicros, TimestampMillis}
-import org.apache.avro.Schema.Type._
-import org.apache.avro.generic._
-import org.apache.avro.util.Utf8
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
-import org.apache.spark.sql.avro.AvroDeserializer.{createDateRebaseFuncInRead, createTimestampRebaseFuncInRead}
-import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.datasources.DataSourceUtils
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
-
-import java.math.BigDecimal
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * A deserializer to deserialize data in avro format to data in catalyst format.
- *
- * NOTE: This code is borrowed from Spark 3.1.2
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[sql] class AvroDeserializer(rootAvroType: Schema,
- rootCatalystType: DataType,
- datetimeRebaseMode: LegacyBehaviorPolicy.Value) {
-
- def this(rootAvroType: Schema, rootCatalystType: DataType) = {
- this(
- rootAvroType,
- rootCatalystType,
- LegacyBehaviorPolicy.withName(SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ)))
- }
-
- private lazy val decimalConversions = new DecimalConversion()
-
- private val dateRebaseFunc = createDateRebaseFuncInRead(
- datetimeRebaseMode, "Avro")
-
- private val timestampRebaseFunc = createTimestampRebaseFuncInRead(
- datetimeRebaseMode, "Avro")
-
- private val converter: Any => Option[Any] = rootCatalystType match {
- // A shortcut for empty schema.
- case st: StructType if st.isEmpty =>
- (data: Any) => Some(InternalRow.empty)
-
- case st: StructType =>
- val resultRow = new SpecificInternalRow(st.map(_.dataType))
- val fieldUpdater = new RowUpdater(resultRow)
- val writer = getRecordWriter(rootAvroType, st, Nil, applyFilters = _ => false)
- (data: Any) => {
- val record = data.asInstanceOf[GenericRecord]
- val skipRow = writer(fieldUpdater, record)
- if (skipRow) None else Some(resultRow)
- }
-
- case _ =>
- val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
- val fieldUpdater = new RowUpdater(tmpRow)
- val writer = newWriter(rootAvroType, rootCatalystType, Nil)
- (data: Any) => {
- writer(fieldUpdater, 0, data)
- Some(tmpRow.get(0, rootCatalystType))
- }
- }
-
- def deserialize(data: Any): Option[Any] = converter(data)
-
- /**
- * Creates a writer to write avro values to Catalyst values at the given ordinal with the given
- * updater.
- */
- private def newWriter(avroType: Schema,
- catalystType: DataType,
- path: List[String]): (CatalystDataUpdater, Int, Any) => Unit =
- (avroType.getType, catalystType) match {
- case (NULL, NullType) => (updater, ordinal, _) =>
- updater.setNullAt(ordinal)
-
- // TODO: we can avoid boxing if future version of avro provide primitive accessors.
- case (BOOLEAN, BooleanType) => (updater, ordinal, value) =>
- updater.setBoolean(ordinal, value.asInstanceOf[Boolean])
-
- case (INT, IntegerType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, value.asInstanceOf[Int])
-
- case (INT, DateType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int]))
-
- case (LONG, LongType) => (updater, ordinal, value) =>
- updater.setLong(ordinal, value.asInstanceOf[Long])
-
- case (LONG, TimestampType) => avroType.getLogicalType match {
- // For backward compatibility, if the Avro type is Long and it is not logical type
- // (the `null` case), the value is processed as timestamp type with millisecond precision.
- case null | _: TimestampMillis => (updater, ordinal, value) =>
- val millis = value.asInstanceOf[Long]
- val micros = AvroUtils.millisToMicros(millis)
- updater.setLong(ordinal, timestampRebaseFunc(micros))
- case _: TimestampMicros => (updater, ordinal, value) =>
- val micros = value.asInstanceOf[Long]
- updater.setLong(ordinal, timestampRebaseFunc(micros))
- case other => throw new IncompatibleSchemaException(
- s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
- }
-
- // Before we upgrade Avro to 1.8 for logical type support, spark-avro converts Long to Date.
- // For backward compatibility, we still keep this conversion.
- case (LONG, DateType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, (value.asInstanceOf[Long] / MILLIS_PER_DAY).toInt)
-
- case (FLOAT, FloatType) => (updater, ordinal, value) =>
- updater.setFloat(ordinal, value.asInstanceOf[Float])
-
- case (DOUBLE, DoubleType) => (updater, ordinal, value) =>
- updater.setDouble(ordinal, value.asInstanceOf[Double])
-
- case (STRING, StringType) => (updater, ordinal, value) =>
- val str = value match {
- case s: String => UTF8String.fromString(s)
- case s: Utf8 =>
- val bytes = new Array[Byte](s.getByteLength)
- System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
- UTF8String.fromBytes(bytes)
- case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
- }
- updater.set(ordinal, str)
-
- case (ENUM, StringType) => (updater, ordinal, value) =>
- updater.set(ordinal, UTF8String.fromString(value.toString))
-
- case (FIXED, BinaryType) => (updater, ordinal, value) =>
- updater.set(ordinal, value.asInstanceOf[GenericFixed].bytes().clone())
-
- case (BYTES, BinaryType) => (updater, ordinal, value) =>
- val bytes = value match {
- case b: ByteBuffer =>
- val bytes = new Array[Byte](b.remaining)
- b.get(bytes)
- // Do not forget to reset the position
- b.rewind()
- bytes
- case b: Array[Byte] => b
- case other => throw new RuntimeException(s"$other is not a valid avro binary.")
- }
- updater.set(ordinal, bytes)
-
- case (FIXED, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
- val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d)
- val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
- updater.setDecimal(ordinal, decimal)
-
- case (BYTES, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
- val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d)
- val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
- updater.setDecimal(ordinal, decimal)
-
- case (RECORD, st: StructType) =>
- // Avro datasource doesn't accept filters with nested attributes. See SPARK-32328.
- // We can always return `false` from `applyFilters` for nested records.
- val writeRecord = getRecordWriter(avroType, st, path, applyFilters = _ => false)
- (updater, ordinal, value) =>
- val row = new SpecificInternalRow(st)
- writeRecord(new RowUpdater(row), value.asInstanceOf[GenericRecord])
- updater.set(ordinal, row)
-
- case (ARRAY, ArrayType(elementType, containsNull)) =>
- val elementWriter = newWriter(avroType.getElementType, elementType, path)
- (updater, ordinal, value) =>
- val collection = value.asInstanceOf[java.util.Collection[Any]]
- val result = createArrayData(elementType, collection.size())
- val elementUpdater = new ArrayDataUpdater(result)
-
- var i = 0
- val iter = collection.iterator()
- while (iter.hasNext) {
- val element = iter.next()
- if (element == null) {
- if (!containsNull) {
- throw new RuntimeException(s"Array value at path ${path.mkString(".")} is not " +
- "allowed to be null")
- } else {
- elementUpdater.setNullAt(i)
- }
- } else {
- elementWriter(elementUpdater, i, element)
- }
- i += 1
- }
-
- updater.set(ordinal, result)
-
- case (MAP, MapType(keyType, valueType, valueContainsNull)) if keyType == StringType =>
- val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType, path)
- val valueWriter = newWriter(avroType.getValueType, valueType, path)
- (updater, ordinal, value) =>
- val map = value.asInstanceOf[java.util.Map[AnyRef, AnyRef]]
- val keyArray = createArrayData(keyType, map.size())
- val keyUpdater = new ArrayDataUpdater(keyArray)
- val valueArray = createArrayData(valueType, map.size())
- val valueUpdater = new ArrayDataUpdater(valueArray)
- val iter = map.entrySet().iterator()
- var i = 0
- while (iter.hasNext) {
- val entry = iter.next()
- assert(entry.getKey != null)
- keyWriter(keyUpdater, i, entry.getKey)
- if (entry.getValue == null) {
- if (!valueContainsNull) {
- throw new RuntimeException(s"Map value at path ${path.mkString(".")} is not " +
- "allowed to be null")
- } else {
- valueUpdater.setNullAt(i)
- }
- } else {
- valueWriter(valueUpdater, i, entry.getValue)
- }
- i += 1
- }
-
- // The Avro map will never have null or duplicated map keys, it's safe to create a
- // ArrayBasedMapData directly here.
- updater.set(ordinal, new ArrayBasedMapData(keyArray, valueArray))
-
- case (UNION, _) =>
- val allTypes = avroType.getTypes.asScala
- val nonNullTypes = allTypes.filter(_.getType != NULL)
- val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava)
- if (nonNullTypes.nonEmpty) {
- if (nonNullTypes.length == 1) {
- newWriter(nonNullTypes.head, catalystType, path)
- } else {
- nonNullTypes.map(_.getType).toSeq match {
- case Seq(a, b) if Set(a, b) == Set(INT, LONG) && catalystType == LongType =>
- (updater, ordinal, value) =>
- value match {
- case null => updater.setNullAt(ordinal)
- case l: java.lang.Long => updater.setLong(ordinal, l)
- case i: java.lang.Integer => updater.setLong(ordinal, i.longValue())
- }
-
- case Seq(a, b) if Set(a, b) == Set(FLOAT, DOUBLE) && catalystType == DoubleType =>
- (updater, ordinal, value) =>
- value match {
- case null => updater.setNullAt(ordinal)
- case d: java.lang.Double => updater.setDouble(ordinal, d)
- case f: java.lang.Float => updater.setDouble(ordinal, f.doubleValue())
- }
-
- case _ =>
- catalystType match {
- case st: StructType if st.length == nonNullTypes.size =>
- val fieldWriters = nonNullTypes.zip(st.fields).map {
- case (schema, field) => newWriter(schema, field.dataType, path :+ field.name)
- }.toArray
- (updater, ordinal, value) => {
- val row = new SpecificInternalRow(st)
- val fieldUpdater = new RowUpdater(row)
- val i = GenericData.get().resolveUnion(nonNullAvroType, value)
- fieldWriters(i)(fieldUpdater, i, value)
- updater.set(ordinal, row)
- }
-
- case _ =>
- throw new IncompatibleSchemaException(
- s"Cannot convert Avro to catalyst because schema at path " +
- s"${path.mkString(".")} is not compatible " +
- s"(avroType = $avroType, sqlType = $catalystType).\n" +
- s"Source Avro schema: $rootAvroType.\n" +
- s"Target Catalyst type: $rootCatalystType")
- }
- }
- }
- } else {
- (updater, ordinal, value) => updater.setNullAt(ordinal)
- }
-
- case _ =>
- throw new IncompatibleSchemaException(
- s"Cannot convert Avro to catalyst because schema at path ${path.mkString(".")} " +
- s"is not compatible (avroType = $avroType, sqlType = $catalystType).\n" +
- s"Source Avro schema: $rootAvroType.\n" +
- s"Target Catalyst type: $rootCatalystType")
- }
-
- // TODO: move the following method in Decimal object on creating Decimal from BigDecimal?
- private def createDecimal(decimal: BigDecimal, precision: Int, scale: Int): Decimal = {
- if (precision <= Decimal.MAX_LONG_DIGITS) {
- // Constructs a `Decimal` with an unscaled `Long` value if possible.
- Decimal(decimal.unscaledValue().longValue(), precision, scale)
- } else {
- // Otherwise, resorts to an unscaled `BigInteger` instead.
- Decimal(decimal, precision, scale)
- }
- }
-
- private def getRecordWriter(avroType: Schema,
- sqlType: StructType,
- path: List[String],
- applyFilters: Int => Boolean): (CatalystDataUpdater, GenericRecord) => Boolean = {
- val validFieldIndexes = ArrayBuffer.empty[Int]
- val fieldWriters = ArrayBuffer.empty[(CatalystDataUpdater, Any) => Unit]
-
- val avroSchemaHelper = new AvroUtils.AvroSchemaHelper(avroType)
- val length = sqlType.length
- var i = 0
- while (i < length) {
- val sqlField = sqlType.fields(i)
- avroSchemaHelper.getFieldByName(sqlField.name) match {
- case Some(avroField) =>
- validFieldIndexes += avroField.pos()
-
- val baseWriter = newWriter(avroField.schema(), sqlField.dataType, path :+ sqlField.name)
- val ordinal = i
- val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => {
- if (value == null) {
- fieldUpdater.setNullAt(ordinal)
- } else {
- baseWriter(fieldUpdater, ordinal, value)
- }
- }
- fieldWriters += fieldWriter
- case None if !sqlField.nullable =>
- val fieldStr = s"${path.mkString(".")}.${sqlField.name}"
- throw new IncompatibleSchemaException(
- s"""
- |Cannot find non-nullable field $fieldStr in Avro schema.
- |Source Avro schema: $rootAvroType.
- |Target Catalyst type: $rootCatalystType.
- """.stripMargin)
- case _ => // nothing to do
- }
- i += 1
- }
-
- (fieldUpdater, record) => {
- var i = 0
- var skipRow = false
- while (i < validFieldIndexes.length && !skipRow) {
- fieldWriters(i)(fieldUpdater, record.get(validFieldIndexes(i)))
- skipRow = applyFilters(i)
- i += 1
- }
- skipRow
- }
- }
-
- private def createArrayData(elementType: DataType, length: Int): ArrayData = elementType match {
- case BooleanType => UnsafeArrayData.fromPrimitiveArray(new Array[Boolean](length))
- case ByteType => UnsafeArrayData.fromPrimitiveArray(new Array[Byte](length))
- case ShortType => UnsafeArrayData.fromPrimitiveArray(new Array[Short](length))
- case IntegerType => UnsafeArrayData.fromPrimitiveArray(new Array[Int](length))
- case LongType => UnsafeArrayData.fromPrimitiveArray(new Array[Long](length))
- case FloatType => UnsafeArrayData.fromPrimitiveArray(new Array[Float](length))
- case DoubleType => UnsafeArrayData.fromPrimitiveArray(new Array[Double](length))
- case _ => new GenericArrayData(new Array[Any](length))
- }
-
- /**
- * A base interface for updating values inside catalyst data structure like `InternalRow` and
- * `ArrayData`.
- */
- sealed trait CatalystDataUpdater {
- def set(ordinal: Int, value: Any): Unit
-
- def setNullAt(ordinal: Int): Unit = set(ordinal, null)
-
- def setBoolean(ordinal: Int, value: Boolean): Unit = set(ordinal, value)
-
- def setByte(ordinal: Int, value: Byte): Unit = set(ordinal, value)
-
- def setShort(ordinal: Int, value: Short): Unit = set(ordinal, value)
-
- def setInt(ordinal: Int, value: Int): Unit = set(ordinal, value)
-
- def setLong(ordinal: Int, value: Long): Unit = set(ordinal, value)
-
- def setDouble(ordinal: Int, value: Double): Unit = set(ordinal, value)
-
- def setFloat(ordinal: Int, value: Float): Unit = set(ordinal, value)
-
- def setDecimal(ordinal: Int, value: Decimal): Unit = set(ordinal, value)
- }
-
- final class RowUpdater(row: InternalRow) extends CatalystDataUpdater {
- override def set(ordinal: Int, value: Any): Unit = row.update(ordinal, value)
-
- override def setNullAt(ordinal: Int): Unit = row.setNullAt(ordinal)
-
- override def setBoolean(ordinal: Int, value: Boolean): Unit = row.setBoolean(ordinal, value)
-
- override def setByte(ordinal: Int, value: Byte): Unit = row.setByte(ordinal, value)
-
- override def setShort(ordinal: Int, value: Short): Unit = row.setShort(ordinal, value)
-
- override def setInt(ordinal: Int, value: Int): Unit = row.setInt(ordinal, value)
-
- override def setLong(ordinal: Int, value: Long): Unit = row.setLong(ordinal, value)
-
- override def setDouble(ordinal: Int, value: Double): Unit = row.setDouble(ordinal, value)
-
- override def setFloat(ordinal: Int, value: Float): Unit = row.setFloat(ordinal, value)
-
- override def setDecimal(ordinal: Int, value: Decimal): Unit =
- row.setDecimal(ordinal, value, value.precision)
- }
-
- final class ArrayDataUpdater(array: ArrayData) extends CatalystDataUpdater {
- override def set(ordinal: Int, value: Any): Unit = array.update(ordinal, value)
-
- override def setNullAt(ordinal: Int): Unit = array.setNullAt(ordinal)
-
- override def setBoolean(ordinal: Int, value: Boolean): Unit = array.setBoolean(ordinal, value)
-
- override def setByte(ordinal: Int, value: Byte): Unit = array.setByte(ordinal, value)
-
- override def setShort(ordinal: Int, value: Short): Unit = array.setShort(ordinal, value)
-
- override def setInt(ordinal: Int, value: Int): Unit = array.setInt(ordinal, value)
-
- override def setLong(ordinal: Int, value: Long): Unit = array.setLong(ordinal, value)
-
- override def setDouble(ordinal: Int, value: Double): Unit = array.setDouble(ordinal, value)
-
- override def setFloat(ordinal: Int, value: Float): Unit = array.setFloat(ordinal, value)
-
- override def setDecimal(ordinal: Int, value: Decimal): Unit = array.update(ordinal, value)
- }
-}
-
-object AvroDeserializer {
-
- // NOTE: Following methods have been renamed in Spark 3.1.3 [1] making [[AvroDeserializer]] implementation
- // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]].
- // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.1.x branch,
- // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.1.2 as well as
- // w/ Spark >= 3.1.3
- //
- // [1] https://github.com/apache/spark/pull/34978
-
- def createDateRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Int => Int = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
- if (days < RebaseDateTime.lastSwitchJulianDay) {
- throw DataSourceUtils.newRebaseExceptionInRead(format)
- }
- days
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays
- case LegacyBehaviorPolicy.CORRECTED => identity[Int]
- }
-
- def createTimestampRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Long => Long = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
- if (micros < RebaseDateTime.lastSwitchJulianTs) {
- throw DataSourceUtils.newRebaseExceptionInRead(format)
- }
- micros
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianMicros
- case LegacyBehaviorPolicy.CORRECTED => identity[Long]
- }
-}
-
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
deleted file mode 100644
index fef0e35ee7b..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Conversions.DecimalConversion
-import org.apache.avro.LogicalTypes.{TimestampMicros, TimestampMillis}
-import org.apache.avro.{LogicalTypes, Schema}
-import org.apache.avro.Schema.Type
-import org.apache.avro.Schema.Type._
-import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record}
-import org.apache.avro.util.Utf8
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.avro.AvroSerializer.{createDateRebaseFuncInWrite, createTimestampRebaseFuncInWrite}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
-import org.apache.spark.sql.catalyst.util.RebaseDateTime
-import org.apache.spark.sql.execution.datasources.DataSourceUtils
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types._
-
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-
-/**
- * A serializer to serialize data in catalyst format to data in avro format.
- *
- * NOTE: This code is borrowed from Spark 3.1.2
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[sql] class AvroSerializer(
- rootCatalystType: DataType,
- rootAvroType: Schema,
- nullable: Boolean,
- datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging {
-
- def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) = {
- this(rootCatalystType, rootAvroType, nullable,
- LegacyBehaviorPolicy.withName(SQLConf.get.getConf(
- SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE)))
- }
-
- def serialize(catalystData: Any): Any = {
- converter.apply(catalystData)
- }
-
- private val dateRebaseFunc = createDateRebaseFuncInWrite(
- datetimeRebaseMode, "Avro")
-
- private val timestampRebaseFunc = createTimestampRebaseFuncInWrite(
- datetimeRebaseMode, "Avro")
-
- private val converter: Any => Any = {
- val actualAvroType = resolveNullableType(rootAvroType, nullable)
- val baseConverter = rootCatalystType match {
- case st: StructType =>
- newStructConverter(st, actualAvroType).asInstanceOf[Any => Any]
- case _ =>
- val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
- val converter = newConverter(rootCatalystType, actualAvroType)
- (data: Any) =>
- tmpRow.update(0, data)
- converter.apply(tmpRow, 0)
- }
- if (nullable) {
- (data: Any) =>
- if (data == null) {
- null
- } else {
- baseConverter.apply(data)
- }
- } else {
- baseConverter
- }
- }
-
- private type Converter = (SpecializedGetters, Int) => Any
-
- private lazy val decimalConversions = new DecimalConversion()
-
- private def newConverter(catalystType: DataType, avroType: Schema): Converter = {
- (catalystType, avroType.getType) match {
- case (NullType, NULL) =>
- (getter, ordinal) => null
- case (BooleanType, BOOLEAN) =>
- (getter, ordinal) => getter.getBoolean(ordinal)
- case (ByteType, INT) =>
- (getter, ordinal) => getter.getByte(ordinal).toInt
- case (ShortType, INT) =>
- (getter, ordinal) => getter.getShort(ordinal).toInt
- case (IntegerType, INT) =>
- (getter, ordinal) => getter.getInt(ordinal)
- case (LongType, LONG) =>
- (getter, ordinal) => getter.getLong(ordinal)
- case (FloatType, FLOAT) =>
- (getter, ordinal) => getter.getFloat(ordinal)
- case (DoubleType, DOUBLE) =>
- (getter, ordinal) => getter.getDouble(ordinal)
- case (d: DecimalType, FIXED)
- if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) =>
- (getter, ordinal) =>
- val decimal = getter.getDecimal(ordinal, d.precision, d.scale)
- decimalConversions.toFixed(decimal.toJavaBigDecimal, avroType,
- LogicalTypes.decimal(d.precision, d.scale))
-
- case (d: DecimalType, BYTES)
- if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) =>
- (getter, ordinal) =>
- val decimal = getter.getDecimal(ordinal, d.precision, d.scale)
- decimalConversions.toBytes(decimal.toJavaBigDecimal, avroType,
- LogicalTypes.decimal(d.precision, d.scale))
-
- case (StringType, ENUM) =>
- val enumSymbols: Set[String] = avroType.getEnumSymbols.asScala.toSet
- (getter, ordinal) =>
- val data = getter.getUTF8String(ordinal).toString
- if (!enumSymbols.contains(data)) {
- throw new IncompatibleSchemaException(
- "Cannot write \"" + data + "\" since it's not defined in enum \"" +
- enumSymbols.mkString("\", \"") + "\"")
- }
- new EnumSymbol(avroType, data)
-
- case (StringType, STRING) =>
- (getter, ordinal) => new Utf8(getter.getUTF8String(ordinal).getBytes)
-
- case (BinaryType, FIXED) =>
- val size = avroType.getFixedSize()
- (getter, ordinal) =>
- val data: Array[Byte] = getter.getBinary(ordinal)
- if (data.length != size) {
- throw new IncompatibleSchemaException(
- s"Cannot write ${data.length} ${if (data.length > 1) "bytes" else "byte"} of " +
- "binary data into FIXED Type with size of " +
- s"$size ${if (size > 1) "bytes" else "byte"}")
- }
- new Fixed(avroType, data)
-
- case (BinaryType, BYTES) =>
- (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
-
- case (DateType, INT) =>
- (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal))
-
- case (TimestampType, LONG) => avroType.getLogicalType match {
- // For backward compatibility, if the Avro type is Long and it is not logical type
- // (the `null` case), output the timestamp value as with millisecond precision.
- case null | _: TimestampMillis => (getter, ordinal) =>
- AvroUtils.microsToMillis(timestampRebaseFunc(getter.getLong(ordinal)))
- case _: TimestampMicros => (getter, ordinal) =>
- timestampRebaseFunc(getter.getLong(ordinal))
- case other => throw new IncompatibleSchemaException(
- s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
- }
-
- case (ArrayType(et, containsNull), ARRAY) =>
- val elementConverter = newConverter(
- et, resolveNullableType(avroType.getElementType, containsNull))
- (getter, ordinal) => {
- val arrayData = getter.getArray(ordinal)
- val len = arrayData.numElements()
- val result = new Array[Any](len)
- var i = 0
- while (i < len) {
- if (containsNull && arrayData.isNullAt(i)) {
- result(i) = null
- } else {
- result(i) = elementConverter(arrayData, i)
- }
- i += 1
- }
- // avro writer is expecting a Java Collection, so we convert it into
- // `ArrayList` backed by the specified array without data copying.
- java.util.Arrays.asList(result: _*)
- }
-
- case (st: StructType, RECORD) =>
- val structConverter = newStructConverter(st, avroType)
- val numFields = st.length
- (getter, ordinal) => structConverter(getter.getStruct(ordinal, numFields))
-
- case (st: StructType, UNION) =>
- val unionConverter = newUnionConverter(st, avroType)
- val numFields = st.length
- (getter, ordinal) => unionConverter(getter.getStruct(ordinal, numFields))
-
- case (MapType(kt, vt, valueContainsNull), MAP) if kt == StringType =>
- val valueConverter = newConverter(
- vt, resolveNullableType(avroType.getValueType, valueContainsNull))
- (getter, ordinal) =>
- val mapData = getter.getMap(ordinal)
- val len = mapData.numElements()
- val result = new java.util.HashMap[String, Any](len)
- val keyArray = mapData.keyArray()
- val valueArray = mapData.valueArray()
- var i = 0
- while (i < len) {
- val key = keyArray.getUTF8String(i).toString
- if (valueContainsNull && valueArray.isNullAt(i)) {
- result.put(key, null)
- } else {
- result.put(key, valueConverter(valueArray, i))
- }
- i += 1
- }
- result
-
- case other =>
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystType to " +
- s"Avro type $avroType.")
- }
- }
-
- private def newStructConverter(catalystStruct: StructType, avroStruct: Schema): InternalRow => Record = {
- if (avroStruct.getType != RECORD || avroStruct.getFields.size() != catalystStruct.length) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " +
- s"Avro type $avroStruct.")
- }
- val avroSchemaHelper = new AvroUtils.AvroSchemaHelper(avroStruct)
-
- val (avroIndices: Array[Int], fieldConverters: Array[Converter]) =
- catalystStruct.map { catalystField =>
- val avroField = avroSchemaHelper.getFieldByName(catalystField.name) match {
- case Some(f) => f
- case None => throw new IncompatibleSchemaException(
- s"Cannot find ${catalystField.name} in Avro schema")
- }
- val converter = newConverter(catalystField.dataType, resolveNullableType(
- avroField.schema(), catalystField.nullable))
- (avroField.pos(), converter)
- }.toArray.unzip
-
- val numFields = catalystStruct.length
- row: InternalRow =>
- val result = new Record(avroStruct)
- var i = 0
- while (i < numFields) {
- if (row.isNullAt(i)) {
- result.put(avroIndices(i), null)
- } else {
- result.put(avroIndices(i), fieldConverters(i).apply(row, i))
- }
- i += 1
- }
- result
- }
-
- private def newUnionConverter(catalystStruct: StructType, avroUnion: Schema): InternalRow => Any = {
- if (avroUnion.getType != UNION || !canMapUnion(catalystStruct, avroUnion)) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " +
- s"Avro type $avroUnion.")
- }
- val nullable = avroUnion.getTypes.size() > 0 && avroUnion.getTypes.get(0).getType == Type.NULL
- val avroInnerTypes = if (nullable) {
- avroUnion.getTypes.asScala.tail
- } else {
- avroUnion.getTypes.asScala
- }
- val fieldConverters = catalystStruct.zip(avroInnerTypes).map {
- case (f1, f2) => newConverter(f1.dataType, f2)
- }
- val numFields = catalystStruct.length
- (row: InternalRow) =>
- var i = 0
- var result: Any = null
- while (i < numFields) {
- if (!row.isNullAt(i)) {
- if (result != null) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " +
- s"Avro union $avroUnion. Record has more than one optional values set")
- }
- result = fieldConverters(i).apply(row, i)
- }
- i += 1
- }
- if (!nullable && result == null) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " +
- s"Avro union $avroUnion. Record has no values set, while should have exactly one")
- }
- result
- }
-
- private def canMapUnion(catalystStruct: StructType, avroStruct: Schema): Boolean = {
- (avroStruct.getTypes.size() > 0 &&
- avroStruct.getTypes.get(0).getType == Type.NULL &&
- avroStruct.getTypes.size() - 1 == catalystStruct.length) || avroStruct.getTypes.size() == catalystStruct.length
- }
-
- /**
- * Resolve a possibly nullable Avro Type.
- *
- * An Avro type is nullable when it is a [[UNION]] of two types: one null type and another
- * non-null type. This method will check the nullability of the input Avro type and return the
- * non-null type within when it is nullable. Otherwise it will return the input Avro type
- * unchanged. It will throw an [[UnsupportedAvroTypeException]] when the input Avro type is an
- * unsupported nullable type.
- *
- * It will also log a warning message if the nullability for Avro and catalyst types are
- * different.
- */
- private def resolveNullableType(avroType: Schema, nullable: Boolean): Schema = {
- val (avroNullable, resolvedAvroType) = resolveAvroType(avroType)
- warnNullabilityDifference(avroNullable, nullable)
- resolvedAvroType
- }
-
- /**
- * Check the nullability of the input Avro type and resolve it when it is nullable. The first
- * return value is a [[Boolean]] indicating if the input Avro type is nullable. The second
- * return value is the possibly resolved type.
- */
- private def resolveAvroType(avroType: Schema): (Boolean, Schema) = {
- if (avroType.getType == Type.UNION) {
- val fields = avroType.getTypes.asScala
- val actualType = fields.filter(_.getType != Type.NULL)
- if (fields.length == 2 && actualType.length == 1) {
- (true, actualType.head)
- } else {
- // This is just a normal union, not used to designate nullability
- (false, avroType)
- }
- } else {
- (false, avroType)
- }
- }
-
- /**
- * log a warning message if the nullability for Avro and catalyst types are different.
- */
- private def warnNullabilityDifference(avroNullable: Boolean, catalystNullable: Boolean): Unit = {
- if (avroNullable && !catalystNullable) {
- logWarning("Writing Avro files with nullable Avro schema and non-nullable catalyst schema.")
- }
- if (!avroNullable && catalystNullable) {
- logWarning("Writing Avro files with non-nullable Avro schema and nullable catalyst " +
- "schema will throw runtime exception if there is a record with null value.")
- }
- }
-}
-
-object AvroSerializer {
-
- // NOTE: Following methods have been renamed in Spark 3.1.3 [1] making [[AvroDeserializer]] implementation
- // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]].
- // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.1.x branch,
- // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.1.2 as well as
- // w/ Spark >= 3.1.3
- //
- // [1] https://github.com/apache/spark/pull/34978
-
- def createDateRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Int => Int = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
- if (days < RebaseDateTime.lastSwitchGregorianDay) {
- throw DataSourceUtils.newRebaseExceptionInWrite(format)
- }
- days
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays
- case LegacyBehaviorPolicy.CORRECTED => identity[Int]
- }
-
- def createTimestampRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Long => Long = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
- if (micros < RebaseDateTime.lastSwitchGregorianTs) {
- throw DataSourceUtils.newRebaseExceptionInWrite(format)
- }
- micros
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
- case LegacyBehaviorPolicy.CORRECTED => identity[Long]
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
deleted file mode 100644
index 77485a1143c..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.util.DateTimeConstants
-import org.apache.spark.sql.internal.SQLConf
-
-import java.util.Locale
-import scala.collection.JavaConverters._
-
-/**
- * NOTE: This code is borrowed from Spark 3.1.3
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[avro] object AvroUtils extends Logging {
-
- /**
- * Wraps an Avro Schema object so that field lookups are faster.
- *
- * @param avroSchema The schema in which to search for fields. Must be of type RECORD.
- */
- class AvroSchemaHelper(avroSchema: Schema) {
- if (avroSchema.getType != Schema.Type.RECORD) {
- throw new IncompatibleSchemaException(
- s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}")
- }
-
- private[this] val fieldMap = avroSchema.getFields.asScala
- .groupBy(_.name.toLowerCase(Locale.ROOT))
- .mapValues(_.toSeq) // toSeq needed for scala 2.13
-
- /**
- * Extract a single field from the contained avro schema which has the desired field name,
- * performing the matching with proper case sensitivity according to SQLConf.resolver.
- *
- * @param name The name of the field to search for.
- * @return `Some(match)` if a matching Avro field is found, otherwise `None`.
- */
- def getFieldByName(name: String): Option[Schema.Field] = {
-
- // get candidates, ignoring case of field name
- val candidates = fieldMap.get(name.toLowerCase(Locale.ROOT))
- .getOrElse(Seq.empty[Schema.Field])
-
- // search candidates, taking into account case sensitivity settings
- candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match {
- case Seq(avroField) => Some(avroField)
- case Seq() => None
- case matches => throw new IncompatibleSchemaException(
- s"Searching for '$name' in Avro schema gave ${matches.size} matches. Candidates: " +
- matches.map(_.name()).mkString("[", ", ", "]")
- )
- }
- }
- }
-
- /**
- * Extract a single field from `avroSchema` which has the desired field name,
- * performing the matching with proper case sensitivity according to [[SQLConf.resolver]].
- *
- * @param avroSchema The schema in which to search for the field. Must be of type RECORD.
- * @param name The name of the field to search for.
- * @return `Some(match)` if a matching Avro field is found, otherwise `None`.
- * @throws IncompatibleSchemaException if `avroSchema` is not a RECORD or contains multiple
- * fields matching `name` (i.e., case-insensitive matching
- * is used and `avroSchema` has two or more fields that have
- * the same name with difference case).
- */
- private[avro] def getAvroFieldByName(
- avroSchema: Schema,
- name: String): Option[Schema.Field] = {
- if (avroSchema.getType != Schema.Type.RECORD) {
- throw new IncompatibleSchemaException(
- s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}")
- }
- avroSchema.getFields.asScala.filter(f => SQLConf.get.resolver(f.name(), name)).toSeq match {
- case Seq(avroField) => Some(avroField)
- case Seq() => None
- case matches => throw new IncompatibleSchemaException(
- s"Searching for '$name' in Avro schema gave ${matches.size} matches. Candidates: " +
- matches.map(_.name()).mkString("[", ", ", "]")
- )
- }
- }
-
- def microsToMillis(micros: Long): Long = {
- // When the timestamp is negative i.e before 1970, we need to adjust the millseconds portion.
- // Example - 1965-01-01 10:11:12.123456 is represented as (-157700927876544) in micro precision.
- // In millis precision the above needs to be represented as (-157700927877).
- Math.floorDiv(micros, DateTimeConstants.MICROS_PER_MILLIS)
- }
-
- def millisToMicros(millis: Long): Long = {
- Math.multiplyExact(millis, DateTimeConstants.MICROS_PER_MILLIS)
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroDeserializer.scala
deleted file mode 100644
index f299e306fe3..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroDeserializer.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types.DataType
-
-class HoodieSpark3_0AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType)
- extends HoodieAvroDeserializer {
-
- private val avroDeserializer = {
- val avroRebaseModeInRead = LegacyBehaviorPolicy.withName(SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
- new AvroDeserializer(rootAvroType, rootCatalystType, avroRebaseModeInRead)
- }
-
- def deserialize(data: Any): Option[Any] = avroDeserializer.deserialize(data)
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroSerializer.scala
deleted file mode 100644
index adae3e99dd4..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroSerializer.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.sql.types.DataType
-
-class HoodieSpark3_0AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean)
- extends HoodieAvroSerializer {
-
- val avroSerializer = new AvroSerializer(rootCatalystType, rootAvroType, nullable)
-
- override def serialize(catalystData: Any): Any = avroSerializer.serialize(catalystData)
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark30Analysis.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark30Analysis.scala
deleted file mode 100644
index 4f143bfada5..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark30Analysis.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolveLambdaVariables, UnresolvedAttribute, UnresolvedExtractValue, caseInsensitiveResolution, withPosition}
-import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentDate, CurrentTimestamp, Expression, ExtractValue, GetStructField, LambdaFunction}
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.util.toPrettySQL
-import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
-
-/**
- * NOTE: Taken from HoodieSpark2Analysis applied to Spark version 3.0.3 and modified to resolve source and target tables
- * if not already resolved
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-object HoodieSpark30Analysis {
-
- case class ResolveReferences(spark: SparkSession) extends Rule[LogicalPlan] {
-
- private val resolver = spark.sessionState.conf.resolver
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
- case mO @ MergeIntoTable(targetTableO, sourceTableO, _, _, _)
- // START: custom Hudi change: don't want to go to the spark mit resolution so we resolve the source and target if they haven't been
- if !mO.resolved || containsUnresolvedStarAssignments(mO) =>
- lazy val analyzer = spark.sessionState.analyzer
- val targetTable = if (targetTableO.resolved) targetTableO else analyzer.execute(targetTableO)
- val sourceTable = if (sourceTableO.resolved) sourceTableO else analyzer.execute(sourceTableO)
- val m = mO.copy(targetTable = targetTable, sourceTable = sourceTable)
- // END: custom Hudi change
- EliminateSubqueryAliases(targetTable) match {
- case _ =>
- val newMatchedActions = m.matchedActions.map {
- case DeleteAction(deleteCondition) =>
- val resolvedDeleteCondition = deleteCondition.map(resolveExpressionTopDown(_, m))
- DeleteAction(resolvedDeleteCondition)
- case UpdateAction(updateCondition, assignments) =>
- val resolvedUpdateCondition = updateCondition.map(resolveExpressionTopDown(_, m))
- // The update value can access columns from both target and source tables.
- UpdateAction(
- resolvedUpdateCondition,
- resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false))
- case o => o
- }
- val newNotMatchedActions = m.notMatchedActions.map {
- case InsertAction(insertCondition, assignments) =>
- // The insert action is used when not matched, so its condition and value can only
- // access columns from the source table.
- val resolvedInsertCondition =
- insertCondition.map(resolveExpressionTopDown(_, Project(Nil, m.sourceTable)))
- InsertAction(
- resolvedInsertCondition,
- resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true))
- case o => o
- }
- val resolvedMergeCondition = resolveExpressionTopDown(m.mergeCondition, m)
- m.copy(mergeCondition = resolvedMergeCondition,
- matchedActions = newMatchedActions,
- notMatchedActions = newNotMatchedActions)
- }
- }
-
- private def resolveAssignments(assignments: Seq[Assignment],
- mergeInto: MergeIntoTable,
- resolveValuesWithSourceOnly: Boolean): Seq[Assignment] = {
- if (assignments.isEmpty) {
- // START: custom Hudi change: filter out meta fields
- val expandedColumns = HoodieSqlCommonUtils.removeMetaFields(mergeInto.targetTable.output)
- // END: custom Hudi change
- val expandedValues = mergeInto.sourceTable.output
- expandedColumns.zip(expandedValues).map(kv => Assignment(kv._1, kv._2))
- } else {
- assignments.map { assign =>
- val resolvedKey = assign.key match {
- case c if !c.resolved =>
- resolveExpressionTopDown(c, Project(Nil, mergeInto.targetTable))
- case o => o
- }
- val resolvedValue = assign.value match {
- // The update values may contain target and/or source references.
- case c if !c.resolved =>
- if (resolveValuesWithSourceOnly) {
- resolveExpressionTopDown(c, Project(Nil, mergeInto.sourceTable))
- } else {
- resolveExpressionTopDown(c, mergeInto)
- }
- case o => o
- }
- Assignment(resolvedKey, resolvedValue)
- }
- }
- }
-
- /**
- * Resolves the attribute and extract value expressions(s) by traversing the
- * input expression in top down manner. The traversal is done in top-down manner as
- * we need to skip over unbound lambda function expression. The lambda expressions are
- * resolved in a different rule [[ResolveLambdaVariables]]
- *
- * Example :
- * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
- *
- * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]]
- *
- * Note : In this routine, the unresolved attributes are resolved from the input plan's
- * children attributes.
- */
- private def resolveExpressionTopDown(e: Expression, q: LogicalPlan): Expression = {
- // scalastyle:off return
- if (e.resolved) return e
- // scalastyle:on return
- e match {
- case f: LambdaFunction if !f.bound => f
- case u@UnresolvedAttribute(nameParts) =>
- // Leave unchanged if resolution fails. Hopefully will be resolved next round.
- val result =
- withPosition(u) {
- q.resolveChildren(nameParts, resolver)
- .orElse(resolveLiteralFunction(nameParts, u, q))
- .getOrElse(u)
- }
- logDebug(s"Resolving $u to $result")
- result
- case UnresolvedExtractValue(child, fieldExpr) if child.resolved =>
- ExtractValue(child, fieldExpr, resolver)
- case _ => e.mapChildren(resolveExpressionTopDown(_, q))
- }
- }
-
- /**
- * Literal functions do not require the user to specify braces when calling them
- * When an attributes is not resolvable, we try to resolve it as a literal function.
- */
- private def resolveLiteralFunction(nameParts: Seq[String],
- attribute: UnresolvedAttribute,
- plan: LogicalPlan): Option[Expression] = {
- // scalastyle:off return
- if (nameParts.length != 1) return None
- // scalastyle:on return
- val isNamedExpression = plan match {
- case Aggregate(_, aggregateExpressions, _) => aggregateExpressions.contains(attribute)
- case Project(projectList, _) => projectList.contains(attribute)
- case Window(windowExpressions, _, _, _) => windowExpressions.contains(attribute)
- case _ => false
- }
- val wrapper: Expression => Expression =
- if (isNamedExpression) f => Alias(f, toPrettySQL(f))() else identity
- // support CURRENT_DATE and CURRENT_TIMESTAMP
- val literalFunctions = Seq(CurrentDate(), CurrentTimestamp())
- val name = nameParts.head
- val func = literalFunctions.find(e => caseInsensitiveResolution(e.prettyName, name))
- func.map(wrapper)
- }
-
- // START: custom Hudi change. Following section is amended to the original (Spark's) implementation
- private def containsUnresolvedStarAssignments(mit: MergeIntoTable): Boolean = {
- val containsUnresolvedInsertStar = mit.notMatchedActions.exists {
- case InsertAction(_, assignments) => assignments.isEmpty
- case _ => false
- }
- val containsUnresolvedUpdateStar = mit.matchedActions.exists {
- case UpdateAction(_, assignments) => assignments.isEmpty
- case _ => false
- }
-
- containsUnresolvedInsertStar || containsUnresolvedUpdateStar
- }
- // END: custom Hudi change.
- }
-
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
deleted file mode 100644
index f78891fa61f..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
-import org.apache.spark.sql.types.DataType
-
-/**
- * ALTER TABLE ... ADD COLUMNS command, as parsed from SQL.
- */
-case class HoodieAlterTableAddColumnsStatement(
- tableName: Seq[String],
- columnsToAdd: Seq[QualifiedColType]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... CHANGE COLUMN command, as parsed from SQL.
- */
-case class HoodieAlterTableAlterColumnStatement(
- tableName: Seq[String],
- column: Seq[String],
- dataType: Option[DataType],
- nullable: Option[Boolean],
- comment: Option[String],
- position: Option[ColumnPosition]) extends ParsedStatement
-
-
-/**
- * ALTER TABLE ... RENAME COLUMN command, as parsed from SQL.
- */
-case class HoodieAlterTableRenameColumnStatement(
- tableName: Seq[String],
- column: Seq[String],
- newName: String) extends ParsedStatement
-
-/**
- * ALTER TABLE ... DROP COLUMNS command, as parsed from SQL.
- */
-case class HoodieAlterTableDropColumnsStatement(
- tableName: Seq[String], columnsToDrop: Seq[Seq[String]]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... SET TBLPROPERTIES command, as parsed from SQL.
- */
-case class HoodieAlterTableSetPropertiesStatement(
- tableName: Seq[String], properties: Map[String, String]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... UNSET TBLPROPERTIES command, as parsed from SQL.
- */
-case class HoodieAlterTableUnsetPropertiesStatement(
- tableName: Seq[String], propertyKeys: Seq[String], ifExists: Boolean) extends ParsedStatement
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala
deleted file mode 100644
index a228d2c8ae9..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources
-
-import org.apache.hudi.storage.StoragePath
-
-import org.apache.hadoop.fs.FileStatus
-import org.apache.spark.sql.catalyst.InternalRow
-
-/**
- * Utils on Spark [[PartitionedFile]] and [[PartitionDirectory]] for Spark 3.0.
- */
-object HoodieSpark30PartitionedFileUtils extends HoodieSparkPartitionedFileUtils {
- override def getPathFromPartitionedFile(partitionedFile: PartitionedFile): StoragePath = {
- new StoragePath(partitionedFile.filePath)
- }
-
- override def getStringPathFromPartitionedFile(partitionedFile: PartitionedFile): String = {
- partitionedFile.filePath
- }
-
- override def createPartitionedFile(partitionValues: InternalRow,
- filePath: StoragePath,
- start: Long,
- length: Long): PartitionedFile = {
- PartitionedFile(partitionValues, filePath.toUri.toString, start, length)
- }
-
- override def toFileStatuses(partitionDirs: Seq[PartitionDirectory]): Seq[FileStatus] = {
- partitionDirs.flatMap(_.files)
- }
-
- override def newPartitionDirectory(internalRow: InternalRow, statuses: Seq[FileStatus]): PartitionDirectory = {
- PartitionDirectory(internalRow, statuses)
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark30NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark30NestedSchemaPruning.scala
deleted file mode 100644
index a8c8e860b9d..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark30NestedSchemaPruning.scala
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources
-
-import org.apache.hudi.{HoodieBaseRelation, SparkAdapterSupport}
-import org.apache.spark.sql.HoodieSpark3CatalystPlanUtils
-import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, AttributeSet, Expression, NamedExpression, ProjectionOverSchema}
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources.BaseRelation
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType}
-
-/**
- * Prunes unnecessary physical columns given a [[PhysicalOperation]] over a data source relation.
- * By "physical column", we mean a column as defined in the data source format like Parquet format
- * or ORC format. For example, in Spark SQL, a root-level Parquet column corresponds to a SQL
- * column, and a nested Parquet column corresponds to a [[StructField]].
- *
- * NOTE: This class is borrowed from Spark 3.2.1, with modifications adapting it to handle [[HoodieBaseRelation]],
- * instead of [[HadoopFsRelation]]
- */
-class Spark30NestedSchemaPruning extends Rule[LogicalPlan] {
- import org.apache.spark.sql.catalyst.expressions.SchemaPruning._
- override def apply(plan: LogicalPlan): LogicalPlan =
- if (SQLConf.get.nestedSchemaPruningEnabled) {
- apply0(plan)
- } else {
- plan
- }
-
- private def apply0(plan: LogicalPlan): LogicalPlan =
- plan transformDown {
- case op @ PhysicalOperation(projects, filters,
- // NOTE: This is modified to accommodate for Hudi's custom relations, given that original
- // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]]
- // TODO generalize to any file-based relation
- l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _))
- if relation.canPruneRelationSchema =>
-
- prunePhysicalColumns(l.output, projects, filters, relation.dataSchema,
- prunedDataSchema => {
- val prunedRelation =
- relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema)
- buildPrunedRelation(l, prunedRelation)
- }).getOrElse(op)
- }
-
- /**
- * This method returns optional logical plan. `None` is returned if no nested field is required or
- * all nested fields are required.
- */
- private def prunePhysicalColumns(output: Seq[AttributeReference],
- projects: Seq[NamedExpression],
- filters: Seq[Expression],
- dataSchema: StructType,
- outputRelationBuilder: StructType => LogicalRelation): Option[LogicalPlan] = {
- val (normalizedProjects, normalizedFilters) =
- normalizeAttributeRefNames(output, projects, filters)
- val requestedRootFields = identifyRootFields(normalizedProjects, normalizedFilters)
-
- // If requestedRootFields includes a nested field, continue. Otherwise,
- // return op
- if (requestedRootFields.exists { root: RootField => !root.derivedFromAtt }) {
- val prunedDataSchema = pruneDataSchema(dataSchema, requestedRootFields)
-
- // If the data schema is different from the pruned data schema, continue. Otherwise,
- // return op. We effect this comparison by counting the number of "leaf" fields in
- // each schemata, assuming the fields in prunedDataSchema are a subset of the fields
- // in dataSchema.
- if (countLeaves(dataSchema) > countLeaves(prunedDataSchema)) {
- val planUtils = SparkAdapterSupport.sparkAdapter.getCatalystPlanUtils.asInstanceOf[HoodieSpark3CatalystPlanUtils]
-
- val prunedRelation = outputRelationBuilder(prunedDataSchema)
- val projectionOverSchema = planUtils.projectOverSchema(prunedDataSchema, AttributeSet(output))
-
- Some(buildNewProjection(projects, normalizedProjects, normalizedFilters,
- prunedRelation, projectionOverSchema))
- } else {
- None
- }
- } else {
- None
- }
- }
-
- /**
- * Normalizes the names of the attribute references in the given projects and filters to reflect
- * the names in the given logical relation. This makes it possible to compare attributes and
- * fields by name. Returns a tuple with the normalized projects and filters, respectively.
- */
- private def normalizeAttributeRefNames(output: Seq[AttributeReference],
- projects: Seq[NamedExpression],
- filters: Seq[Expression]): (Seq[NamedExpression], Seq[Expression]) = {
- val normalizedAttNameMap = output.map(att => (att.exprId, att.name)).toMap
- val normalizedProjects = projects.map(_.transform {
- case att: AttributeReference if normalizedAttNameMap.contains(att.exprId) =>
- att.withName(normalizedAttNameMap(att.exprId))
- }).map { case expr: NamedExpression => expr }
- val normalizedFilters = filters.map(_.transform {
- case att: AttributeReference if normalizedAttNameMap.contains(att.exprId) =>
- att.withName(normalizedAttNameMap(att.exprId))
- })
- (normalizedProjects, normalizedFilters)
- }
-
- /**
- * Builds the new output [[Project]] Spark SQL operator that has the `leafNode`.
- */
- private def buildNewProjection(projects: Seq[NamedExpression],
- normalizedProjects: Seq[NamedExpression],
- filters: Seq[Expression],
- prunedRelation: LogicalRelation,
- projectionOverSchema: ProjectionOverSchema): Project = {
- // Construct a new target for our projection by rewriting and
- // including the original filters where available
- val projectionChild =
- if (filters.nonEmpty) {
- val projectedFilters = filters.map(_.transformDown {
- case projectionOverSchema(expr) => expr
- })
- val newFilterCondition = projectedFilters.reduce(And)
- Filter(newFilterCondition, prunedRelation)
- } else {
- prunedRelation
- }
-
- // Construct the new projections of our Project by
- // rewriting the original projections
- val newProjects = normalizedProjects.map(_.transformDown {
- case projectionOverSchema(expr) => expr
- }).map { case expr: NamedExpression => expr }
-
- if (log.isDebugEnabled) {
- logDebug(s"New projects:\n${newProjects.map(_.treeString).mkString("\n")}")
- }
-
- Project(restoreOriginalOutputNames(newProjects, projects.map(_.name)), projectionChild)
- }
-
- /**
- * Builds a pruned logical relation from the output of the output relation and the schema of the
- * pruned base relation.
- */
- private def buildPrunedRelation(outputRelation: LogicalRelation,
- prunedBaseRelation: BaseRelation): LogicalRelation = {
- val prunedOutput = getPrunedOutput(outputRelation.output, prunedBaseRelation.schema)
- outputRelation.copy(relation = prunedBaseRelation, output = prunedOutput)
- }
-
- // Prune the given output to make it consistent with `requiredSchema`.
- private def getPrunedOutput(output: Seq[AttributeReference],
- requiredSchema: StructType): Seq[AttributeReference] = {
- // We need to replace the expression ids of the pruned relation output attributes
- // with the expression ids of the original relation output attributes so that
- // references to the original relation's output are not broken
- val outputIdMap = output.map(att => (att.name, att.exprId)).toMap
- requiredSchema
- .toAttributes
- .map {
- case att if outputIdMap.contains(att.name) =>
- att.withExprId(outputIdMap(att.name))
- case att => att
- }
- }
-
- /**
- * Counts the "leaf" fields of the given dataType. Informally, this is the
- * number of fields of non-complex data type in the tree representation of
- * [[DataType]].
- */
- private def countLeaves(dataType: DataType): Int = {
- dataType match {
- case array: ArrayType => countLeaves(array.elementType)
- case map: MapType => countLeaves(map.keyType) + countLeaves(map.valueType)
- case struct: StructType =>
- struct.map(field => countLeaves(field.dataType)).sum
- case _ => 1
- }
- }
-
- private def restoreOriginalOutputNames(
- projectList: Seq[NamedExpression],
- originalNames: Seq[String]): Seq[NamedExpression] = {
- projectList.zip(originalNames).map {
- case (attr: Attribute, name) => attr.withName(name)
- case (alias: Alias, name) => if (name == alias.name) {
- alias
- } else {
- AttributeReference(name, alias.dataType, alias.nullable, alias.metadata)(alias.exprId, alias.qualifier)
- }
- case (other, _) => other
- }
- }
-
- // NOTE: `pruneDataSchema` and `sortLeftFieldsByRight` functions are copied from Spark 3.1.2,
- // as these functions in `SchemaPruning` have bugs in Spark 3.0.2 (see SPARK-35096,
- // https://github.com/apache/spark/commit/2bbe0a4151f2af00f1105489d5757be28ff278d6)
- /**
- * Prunes the nested schema by the requested fields. For example, if the schema is:
- * `id int, s struct<a:int, b:int>`, and given requested field "s.a", the inner field "b"
- * is pruned in the returned schema: `id int, s struct<a:int>`.
- * Note that:
- * 1. The schema field ordering at original schema is still preserved in pruned schema.
- * 2. The top-level fields are not pruned here.
- */
- private def pruneDataSchema(
- dataSchema: StructType,
- requestedRootFields: Seq[RootField]): StructType = {
- val resolver = SQLConf.get.resolver
- // Merge the requested root fields into a single schema. Note the ordering of the fields
- // in the resulting schema may differ from their ordering in the logical relation's
- // original schema
- val mergedSchema = requestedRootFields
- .map { root: RootField => StructType(Array(root.field)) }
- .reduceLeft(_ merge _)
- val mergedDataSchema =
- StructType(dataSchema.map(d => mergedSchema.find(m => resolver(m.name, d.name)).getOrElse(d)))
- // Sort the fields of mergedDataSchema according to their order in dataSchema,
- // recursively. This makes mergedDataSchema a pruned schema of dataSchema
- sortLeftFieldsByRight(mergedDataSchema, dataSchema).asInstanceOf[StructType]
- }
-
- /**
- * Sorts the fields and descendant fields of structs in left according to their order in
- * right. This function assumes that the fields of left are a subset of the fields of
- * right, recursively. That is, left is a "subschema" of right, ignoring order of
- * fields.
- */
- private def sortLeftFieldsByRight(left: DataType, right: DataType): DataType =
- (left, right) match {
- case (ArrayType(leftElementType, containsNull), ArrayType(rightElementType, _)) =>
- ArrayType(
- sortLeftFieldsByRight(leftElementType, rightElementType),
- containsNull)
- case (MapType(leftKeyType, leftValueType, containsNull),
- MapType(rightKeyType, rightValueType, _)) =>
- MapType(
- sortLeftFieldsByRight(leftKeyType, rightKeyType),
- sortLeftFieldsByRight(leftValueType, rightValueType),
- containsNull)
- case (leftStruct: StructType, rightStruct: StructType) =>
- val resolver = SQLConf.get.resolver
- val filteredRightFieldNames = rightStruct.fieldNames
- .filter(name => leftStruct.fieldNames.exists(resolver(_, name)))
- val sortedLeftFields = filteredRightFieldNames.map { fieldName =>
- val resolvedLeftStruct = leftStruct.find(p => resolver(p.name, fieldName)).get
- val leftFieldType = resolvedLeftStruct.dataType
- val rightFieldType = rightStruct(fieldName).dataType
- val sortedLeftFieldType = sortLeftFieldsByRight(leftFieldType, rightFieldType)
- StructField(fieldName, sortedLeftFieldType, nullable = resolvedLeftStruct.nullable)
- }
- StructType(sortedLeftFields)
- case _ => left
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
deleted file mode 100644
index 59fde4af02f..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
+++ /dev/null
@@ -1,418 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet
-
-import org.apache.hudi.HoodieSparkUtils
-import org.apache.hudi.client.utils.SparkInternalSchemaConverter
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.common.util.InternalSchemaCache
-import org.apache.hudi.common.util.StringUtils.isNullOrEmpty
-import org.apache.hudi.common.util.collection.Pair
-import org.apache.hudi.internal.schema.InternalSchema
-import org.apache.hudi.internal.schema.action.InternalSchemaMerger
-import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType}
-import org.apache.parquet.filter2.compat.FilterCompat
-import org.apache.parquet.filter2.predicate.FilterApi
-import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
-import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetRecordReader}
-import org.apache.spark.TaskContext
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.avro.AvroDeserializer
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
-import org.apache.spark.sql.catalyst.expressions.{Cast, JoinedRow}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.datasources.parquet.Spark30LegacyHoodieParquetFileFormat.{createParquetFilters, pruneInternalSchema, rebuildFilterFromParquet}
-import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType}
-import org.apache.spark.util.SerializableConfiguration
-
-import java.net.URI
-
-
-/**
- * This class is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior
- * that's not possible to customize in any other way
- *
- * NOTE: This is a version of [[AvroDeserializer]] impl from Spark 3.1.2 w/ w/ the following changes applied to it:
- * <ol>
- * <li>Avoiding appending partition values to the rows read from the data file</li>
- * <li>Schema on-read</li>
- * </ol>
- */
-class Spark30LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValues: Boolean) extends ParquetFileFormat {
-
- override def buildReaderWithPartitionValues(sparkSession: SparkSession,
- dataSchema: StructType,
- partitionSchema: StructType,
- requiredSchema: StructType,
- filters: Seq[Filter],
- options: Map[String, String],
- hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = {
- hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName)
- hadoopConf.set(
- ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA,
- requiredSchema.json)
- hadoopConf.set(
- ParquetWriteSupport.SPARK_ROW_SCHEMA,
- requiredSchema.json)
- hadoopConf.set(
- SQLConf.SESSION_LOCAL_TIMEZONE.key,
- sparkSession.sessionState.conf.sessionLocalTimeZone)
- hadoopConf.setBoolean(
- SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key,
- sparkSession.sessionState.conf.nestedSchemaPruningEnabled)
- hadoopConf.setBoolean(
- SQLConf.CASE_SENSITIVE.key,
- sparkSession.sessionState.conf.caseSensitiveAnalysis)
-
- ParquetWriteSupport.setSchema(requiredSchema, hadoopConf)
-
- // Sets flags for `ParquetToSparkSchemaConverter`
- hadoopConf.setBoolean(
- SQLConf.PARQUET_BINARY_AS_STRING.key,
- sparkSession.sessionState.conf.isParquetBinaryAsString)
- hadoopConf.setBoolean(
- SQLConf.PARQUET_INT96_AS_TIMESTAMP.key,
- sparkSession.sessionState.conf.isParquetINT96AsTimestamp)
-
- val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA)
- // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself,
- // therefore it's safe to do schema projection here
- if (!isNullOrEmpty(internalSchemaStr)) {
- val prunedInternalSchemaStr =
- pruneInternalSchema(internalSchemaStr, requiredSchema)
- hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr)
- }
-
- val broadcastedHadoopConf =
- sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
-
- // TODO: if you move this into the closure it reverts to the default values.
- // If true, enable using the custom RecordReader for parquet. This only works for
- // a subset of the types (no complex types).
- val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields)
- val sqlConf = sparkSession.sessionState.conf
- val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled
- val enableVectorizedReader: Boolean =
- sqlConf.parquetVectorizedReaderEnabled &&
- resultSchema.forall(_.dataType.isInstanceOf[AtomicType])
- val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled
- val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion
- val capacity = sqlConf.parquetVectorizedReaderBatchSize
- val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown
- // Whole stage codegen (PhysicalRDD) is able to deal with batches directly
- val returningBatch = supportBatch(sparkSession, resultSchema)
- val pushDownDate = sqlConf.parquetFilterPushDownDate
- val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp
- val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal
- val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith
- val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold
- val isCaseSensitive = sqlConf.caseSensitiveAnalysis
- val timeZoneId = Option(sqlConf.sessionLocalTimeZone)
-
- (file: PartitionedFile) => {
- assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size)
-
- val filePath = new Path(new URI(file.filePath))
- val split =
- new org.apache.parquet.hadoop.ParquetInputSplit(
- filePath,
- file.start,
- file.start + file.length,
- file.length,
- Array.empty,
- null)
-
- val sharedConf = broadcastedHadoopConf.value.value
-
- // Fetch internal schema
- val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA)
- // Internal schema has to be pruned at this point
- val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr)
-
- var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent
-
- val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH)
- val fileSchema = if (shouldUseInternalSchema) {
- val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
- val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
- val storage = new HoodieHadoopStorage(tablePath, sharedConf)
- InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
- } else {
- null
- }
-
- lazy val footerFileMetaData =
- ParquetFileReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData
- val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
- footerFileMetaData.getKeyValueMetaData.get,
- SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
- // Try to push down filters when filter push-down is enabled.
- val pushed = if (enableParquetFilterPushDown) {
- val parquetSchema = footerFileMetaData.getSchema
- val parquetFilters = if (HoodieSparkUtils.gteqSpark3_1_3) {
- createParquetFilters(
- parquetSchema,
- pushDownDate,
- pushDownTimestamp,
- pushDownDecimal,
- pushDownStringStartWith,
- pushDownInFilterThreshold,
- isCaseSensitive,
- datetimeRebaseMode)
- } else {
- createParquetFilters(
- parquetSchema,
- pushDownDate,
- pushDownTimestamp,
- pushDownDecimal,
- pushDownStringStartWith,
- pushDownInFilterThreshold,
- isCaseSensitive)
- }
- filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null)))
- // Collects all converted Parquet filter predicates. Notice that not all predicates can be
- // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap`
- // is used here.
- .flatMap(parquetFilters.createFilter)
- .reduceOption(FilterApi.and)
- } else {
- None
- }
-
- // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps'
- // *only* if the file was created by something other than "parquet-mr", so check the actual
- // writer here for this file. We have to do this per-file, as each file in the table may
- // have different writers.
- // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads.
- def isCreatedByParquetMr: Boolean =
- footerFileMetaData.getCreatedBy().startsWith("parquet-mr")
-
- val convertTz =
- if (timestampConversion && !isCreatedByParquetMr) {
- Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
- } else {
- None
- }
-
- val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
-
- // Clone new conf
- val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value)
-
- val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) {
- val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema()
- val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema)
-
- hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json)
-
- SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema)
- } else {
- val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema)
- if (!implicitTypeChangeInfo.isEmpty) {
- shouldUseInternalSchema = true
- hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json)
- }
- implicitTypeChangeInfo
- }
-
- val hadoopAttemptContext =
- new TaskAttemptContextImpl(hadoopAttemptConf, attemptId)
-
- // Try to push down filters when filter push-down is enabled.
- // Notice: This push-down is RowGroups level, not individual records.
- if (pushed.isDefined) {
- ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
- }
- val taskContext = Option(TaskContext.get())
- if (enableVectorizedReader) {
- val vectorizedReader =
- if (shouldUseInternalSchema) {
- new Spark30HoodieVectorizedParquetRecordReader(
- convertTz.orNull,
- datetimeRebaseMode.toString,
- enableOffHeapColumnVector && taskContext.isDefined,
- capacity,
- typeChangeInfos)
- } else {
- new VectorizedParquetRecordReader(
- convertTz.orNull,
- datetimeRebaseMode.toString,
- enableOffHeapColumnVector && taskContext.isDefined,
- capacity)
- }
-
- val iter = new RecordReaderIterator(vectorizedReader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- vectorizedReader.initialize(split, hadoopAttemptContext)
-
- // NOTE: We're making appending of the partitioned values to the rows read from the
- // data file configurable
- if (shouldAppendPartitionValues) {
- logDebug(s"Appending $partitionSchema ${file.partitionValues}")
- vectorizedReader.initBatch(partitionSchema, file.partitionValues)
- } else {
- vectorizedReader.initBatch(StructType(Nil), InternalRow.empty)
- }
-
- if (returningBatch) {
- vectorizedReader.enableReturningBatches()
- }
-
- // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy.
- iter.asInstanceOf[Iterator[InternalRow]]
- } else {
- logDebug(s"Falling back to parquet-mr")
- // ParquetRecordReader returns InternalRow
- val readSupport = new ParquetReadSupport(
- convertTz,
- enableVectorizedReader = false,
- datetimeRebaseMode)
- val reader = if (pushed.isDefined && enableRecordFilter) {
- val parquetFilter = FilterCompat.get(pushed.get, null)
- new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
- } else {
- new ParquetRecordReader[InternalRow](readSupport)
- }
- val iter = new RecordReaderIterator[InternalRow](reader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- reader.initialize(split, hadoopAttemptContext)
-
- val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes
- val unsafeProjection = if (typeChangeInfos.isEmpty) {
- GenerateUnsafeProjection.generate(fullSchema, fullSchema)
- } else {
- // find type changed.
- val newFullSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) =>
- if (typeChangeInfos.containsKey(i)) {
- StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata)
- } else f
- }).toAttributes ++ partitionSchema.toAttributes
- val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) =>
- if (typeChangeInfos.containsKey(i)) {
- val srcType = typeChangeInfos.get(i).getRight
- val dstType = typeChangeInfos.get(i).getLeft
- val needTimeZone = Cast.needsTimeZone(srcType, dstType)
- Cast(attr, dstType, if (needTimeZone) timeZoneId else None)
- } else attr
- }
- GenerateUnsafeProjection.generate(castSchema, newFullSchema)
- }
-
- // NOTE: We're making appending of the partitioned values to the rows read from the
- // data file configurable
- if (!shouldAppendPartitionValues || partitionSchema.length == 0) {
- // There is no partition columns
- iter.map(unsafeProjection)
- } else {
- val joinedRow = new JoinedRow()
- iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues)))
- }
- }
- }
- }
-}
-
-object Spark30LegacyHoodieParquetFileFormat {
-
- def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = {
- val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr)
- if (querySchemaOption.isPresent && requiredSchema.nonEmpty) {
- val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get())
- SerDeHelper.toJson(prunedSchema)
- } else {
- internalSchemaStr
- }
- }
-
- private def createParquetFilters(args: Any*): ParquetFilters = {
- // ParquetFilters bears a single ctor (in Spark 3.1)
- val ctor = classOf[ParquetFilters].getConstructors.head
- ctor.newInstance(args.map(_.asInstanceOf[AnyRef]): _*)
- .asInstanceOf[ParquetFilters]
- }
-
- private def rebuildFilterFromParquet(oldFilter: Filter, fileSchema: InternalSchema, querySchema: InternalSchema): Filter = {
- if (fileSchema == null || querySchema == null) {
- oldFilter
- } else {
- oldFilter match {
- case eq: EqualTo =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(eq.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else eq.copy(attribute = newAttribute)
- case eqs: EqualNullSafe =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(eqs.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else eqs.copy(attribute = newAttribute)
- case gt: GreaterThan =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(gt.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else gt.copy(attribute = newAttribute)
- case gtr: GreaterThanOrEqual =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(gtr.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else gtr.copy(attribute = newAttribute)
- case lt: LessThan =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(lt.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else lt.copy(attribute = newAttribute)
- case lte: LessThanOrEqual =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(lte.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else lte.copy(attribute = newAttribute)
- case i: In =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(i.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else i.copy(attribute = newAttribute)
- case isn: IsNull =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(isn.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else isn.copy(attribute = newAttribute)
- case isnn: IsNotNull =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(isnn.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else isnn.copy(attribute = newAttribute)
- case And(left, right) =>
- And(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema))
- case Or(left, right) =>
- Or(rebuildFilterFromParquet(left, fileSchema, querySchema), rebuildFilterFromParquet(right, fileSchema, querySchema))
- case Not(child) =>
- Not(rebuildFilterFromParquet(child, fileSchema, querySchema))
- case ssw: StringStartsWith =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(ssw.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else ssw.copy(attribute = newAttribute)
- case ses: StringEndsWith =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(ses.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else ses.copy(attribute = newAttribute)
- case sc: StringContains =>
- val newAttribute = InternalSchemaUtils.reBuildFilterName(sc.attribute, fileSchema, querySchema)
- if (newAttribute.isEmpty) AlwaysTrue else sc.copy(attribute = newAttribute)
- case AlwaysTrue =>
- AlwaysTrue
- case AlwaysFalse =>
- AlwaysFalse
- case _ =>
- AlwaysTrue
- }
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetReader.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetReader.scala
deleted file mode 100644
index 22042c964bc..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetReader.scala
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet
-
-import org.apache.hudi.common.util
-import org.apache.hudi.internal.schema.InternalSchema
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapreduce._
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.parquet.filter2.compat.FilterCompat
-import org.apache.parquet.filter2.predicate.FilterApi
-import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
-import org.apache.parquet.hadoop._
-import org.apache.spark.TaskContext
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.JoinedRow
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.datasources._
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types._
-
-import java.net.URI
-
-class Spark30ParquetReader(enableVectorizedReader: Boolean,
- enableParquetFilterPushDown: Boolean,
- pushDownDate: Boolean,
- pushDownTimestamp: Boolean,
- pushDownDecimal: Boolean,
- pushDownInFilterThreshold: Int,
- pushDownStringStartWith: Boolean,
- isCaseSensitive: Boolean,
- timestampConversion: Boolean,
- enableOffHeapColumnVector: Boolean,
- capacity: Int,
- returningBatch: Boolean,
- enableRecordFilter: Boolean,
- timeZoneId: Option[String]) extends SparkParquetReaderBase(
- enableVectorizedReader = enableVectorizedReader,
- enableParquetFilterPushDown = enableParquetFilterPushDown,
- pushDownDate = pushDownDate,
- pushDownTimestamp = pushDownTimestamp,
- pushDownDecimal = pushDownDecimal,
- pushDownInFilterThreshold = pushDownInFilterThreshold,
- isCaseSensitive = isCaseSensitive,
- timestampConversion = timestampConversion,
- enableOffHeapColumnVector = enableOffHeapColumnVector,
- capacity = capacity,
- returningBatch = returningBatch,
- enableRecordFilter = enableRecordFilter,
- timeZoneId = timeZoneId) {
-
- /**
- * Read an individual parquet file
- * Code from ParquetFileFormat#buildReaderWithPartitionValues from Spark v3.0.3 adapted here
- *
- * @param file parquet file to read
- * @param requiredSchema desired output schema of the data
- * @param partitionSchema schema of the partition columns. Partition values will be appended to the end of every row
- * @param internalSchemaOpt option of internal schema for schema.on.read
- * @param filters filters for data skipping. Not guaranteed to be used; the spark plan will also apply the filters.
- * @param sharedConf the hadoop conf
- * @return iterator of rows read from the file output type says [[InternalRow]] but could be [[ColumnarBatch]]
- */
- protected def doRead(file: PartitionedFile,
- requiredSchema: StructType,
- partitionSchema: StructType,
- internalSchemaOpt: org.apache.hudi.common.util.Option[InternalSchema],
- filters: Seq[Filter],
- sharedConf: Configuration): Iterator[InternalRow] = {
- assert(file.partitionValues.numFields == partitionSchema.size)
-
- val filePath = new Path(new URI(file.filePath))
- val split =
- new org.apache.parquet.hadoop.ParquetInputSplit(
- filePath,
- file.start,
- file.start + file.length,
- file.length,
- Array.empty,
- null)
-
- val schemaEvolutionUtils = new Spark30ParquetSchemaEvolutionUtils(sharedConf, filePath, requiredSchema,
- partitionSchema, internalSchemaOpt)
-
- lazy val footerFileMetaData =
- ParquetFileReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData
- // Try to push down filters when filter push-down is enabled.
- val pushed = if (enableParquetFilterPushDown) {
- val parquetSchema = footerFileMetaData.getSchema
- val parquetFilters = createParquetFilters(
- parquetSchema,
- pushDownDate,
- pushDownTimestamp,
- pushDownDecimal,
- pushDownStringStartWith,
- pushDownInFilterThreshold,
- isCaseSensitive)
- filters.map(schemaEvolutionUtils.rebuildFilterFromParquet)
- // Collects all converted Parquet filter predicates. Notice that not all predicates can be
- // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap`
- // is used here.
- .flatMap(parquetFilters.createFilter(_))
- .reduceOption(FilterApi.and)
- } else {
- None
- }
-
- // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps'
- // *only* if the file was created by something other than "parquet-mr", so check the actual
- // writer here for this file. We have to do this per-file, as each file in the table may
- // have different writers.
- // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads.
- def isCreatedByParquetMr: Boolean =
- footerFileMetaData.getCreatedBy().startsWith("parquet-mr")
-
- val convertTz =
- if (timestampConversion && !isCreatedByParquetMr) {
- Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
- } else {
- None
- }
-
- val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
- footerFileMetaData.getKeyValueMetaData.get,
- SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
-
- val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
- val hadoopAttemptContext =
- new TaskAttemptContextImpl(schemaEvolutionUtils.getHadoopConfClone(footerFileMetaData, enableVectorizedReader), attemptId)
-
- // Try to push down filters when filter push-down is enabled.
- // Notice: This push-down is RowGroups level, not individual records.
- if (pushed.isDefined) {
- ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
- }
- val taskContext = Option(TaskContext.get())
- if (enableVectorizedReader) {
- val vectorizedReader = schemaEvolutionUtils.buildVectorizedReader(
- convertTz.orNull,
- datetimeRebaseMode.toString,
- enableOffHeapColumnVector && taskContext.isDefined,
- capacity)
- val iter = new RecordReaderIterator(vectorizedReader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- vectorizedReader.initialize(split, hadoopAttemptContext)
- vectorizedReader.initBatch(partitionSchema, file.partitionValues)
- if (returningBatch) {
- vectorizedReader.enableReturningBatches()
- }
-
- // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy.
- iter.asInstanceOf[Iterator[InternalRow]]
- } else {
- // ParquetRecordReader returns InternalRow
- val readSupport = new ParquetReadSupport(
- convertTz, enableVectorizedReader = false, datetimeRebaseMode)
- val reader = if (pushed.isDefined && enableRecordFilter) {
- val parquetFilter = FilterCompat.get(pushed.get, null)
- new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
- } else {
- new ParquetRecordReader[InternalRow](readSupport)
- }
- val iter = new RecordReaderIterator[InternalRow](reader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- reader.initialize(split, hadoopAttemptContext)
-
- val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes
- val unsafeProjection = schemaEvolutionUtils.generateUnsafeProjection(fullSchema, timeZoneId)
-
- if (partitionSchema.length == 0) {
- // There is no partition columns
- iter.map(unsafeProjection)
- } else {
- val joinedRow = new JoinedRow()
- iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues)))
- }
- }
- }
-
- private def createParquetFilters(args: Any*): ParquetFilters = {
- val ctor = classOf[ParquetFilters].getConstructors.head
- ctor.newInstance(args.map(_.asInstanceOf[AnyRef]): _*)
- .asInstanceOf[ParquetFilters]
- }
-}
-
-object Spark30ParquetReader extends SparkParquetReaderBuilder {
- /**
- * Get parquet file reader
- *
- * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc
- * @param sqlConf the [[SQLConf]] used for the read
- * @param options passed as a param to the file format
- * @param hadoopConf some configs will be set for the hadoopConf
- * @return parquet file reader
- */
- def build(vectorized: Boolean,
- sqlConf: SQLConf,
- options: Map[String, String],
- hadoopConf: Configuration): SparkParquetReader = {
- //set hadoopconf
- hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName)
- hadoopConf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, sqlConf.sessionLocalTimeZone)
- hadoopConf.setBoolean(SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, sqlConf.nestedSchemaPruningEnabled)
- hadoopConf.setBoolean(SQLConf.CASE_SENSITIVE.key, sqlConf.caseSensitiveAnalysis)
- hadoopConf.setBoolean(SQLConf.PARQUET_BINARY_AS_STRING.key, sqlConf.isParquetBinaryAsString)
- hadoopConf.setBoolean(SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, sqlConf.isParquetINT96AsTimestamp)
-
- new Spark30ParquetReader(
- enableVectorizedReader = vectorized,
- enableParquetFilterPushDown = sqlConf.parquetFilterPushDown,
- pushDownDate = sqlConf.parquetFilterPushDownDate,
- pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp,
- pushDownDecimal = sqlConf.parquetFilterPushDownDecimal,
- pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold,
- pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith,
- isCaseSensitive = sqlConf.caseSensitiveAnalysis,
- timestampConversion = sqlConf.isParquetINT96TimestampConversion,
- enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled,
- capacity = sqlConf.parquetVectorizedReaderBatchSize,
- returningBatch = sqlConf.parquetVectorizedReaderEnabled,
- enableRecordFilter = sqlConf.parquetRecordFilterEnabled,
- timeZoneId = Some(sqlConf.sessionLocalTimeZone))
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetSchemaEvolutionUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetSchemaEvolutionUtils.scala
deleted file mode 100644
index 2fccf6a7a72..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30ParquetSchemaEvolutionUtils.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet
-
-import org.apache.hudi.common.util
-import org.apache.hudi.internal.schema.InternalSchema
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.execution.datasources.Spark3ParquetSchemaEvolutionUtils
-import org.apache.spark.sql.types.StructType
-
-import java.time.ZoneId
-
-class Spark30ParquetSchemaEvolutionUtils(sharedConf: Configuration,
- filePath: Path,
- requiredSchema: StructType,
- partitionSchema: StructType,
- internalSchemaOpt: util.Option[InternalSchema]) extends
- Spark3ParquetSchemaEvolutionUtils(sharedConf, filePath, requiredSchema, partitionSchema, internalSchemaOpt) {
-
- def buildVectorizedReader(convertTz: ZoneId,
- datetimeRebaseMode: String,
- useOffHeap: Boolean,
- capacity: Int): VectorizedParquetRecordReader = {
- if (shouldUseInternalSchema) {
- new Spark30HoodieVectorizedParquetRecordReader(
- convertTz,
- datetimeRebaseMode,
- useOffHeap,
- capacity,
- typeChangeInfos)
- } else {
- new VectorizedParquetRecordReader(
- convertTz,
- datetimeRebaseMode,
- useOffHeap,
- capacity)
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark30ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark30ResolveHudiAlterTableCommand.scala
deleted file mode 100644
index 593432c5943..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark30ResolveHudiAlterTableCommand.scala
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hudi
-
-import org.apache.hudi.common.config.HoodieCommonConfig
-import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.CatalogTable
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.TableChange._
-import org.apache.spark.sql.connector.catalog.TableChange
-import org.apache.spark.sql.hudi.command.Spark30AlterTableCommand
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructType}
-import org.apache.spark.sql.{AnalysisException, SparkSession}
-
-import java.util.Locale
-import scala.collection.mutable
-
-/**
- * Rule to mostly resolve, normalize and rewrite column names based on case sensitivity
- * for alter table column commands.
- * TODO: we should remove this file when we support datasourceV2 for hoodie on spark3.0.x
- */
-case class Spark30ResolveHudiAlterTableCommand(sparkSession: SparkSession) extends Rule[LogicalPlan] {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
- def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
- case add @ HoodieAlterTableAddColumnsStatement(asTable(table), cols) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled){
- cols.foreach(c => failNullType(c.dataType))
- val changes = cols.map { col =>
- TableChange.addColumn(
- col.name.toArray,
- col.dataType,
- col.nullable,
- col.comment.orNull,
- col.position.orNull)
- }
- val newChanges = normalizeChanges(changes, table.schema)
- Spark30AlterTableCommand(table, newChanges, ColumnChangeID.ADD)
- } else {
- // throw back to spark
- AlterTableAddColumnsStatement(add.tableName, add.columnsToAdd)
- }
- case a @ HoodieAlterTableAlterColumnStatement(asTable(table), _, _, _, _, _) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled){
- a.dataType.foreach(failNullType)
- val colName = a.column.toArray
- val typeChange = a.dataType.map { newDataType =>
- TableChange.updateColumnType(colName, newDataType)
- }
- val nullabilityChange = a.nullable.map { nullable =>
- TableChange.updateColumnNullability(colName, nullable)
- }
- val commentChange = a.comment.map { newComment =>
- TableChange.updateColumnComment(colName, newComment)
- }
- val positionChange = a.position.map { newPosition =>
- TableChange.updateColumnPosition(colName, newPosition)
- }
- Spark30AlterTableCommand(table, normalizeChanges(typeChange.toSeq ++ nullabilityChange ++ commentChange ++ positionChange, table.schema), ColumnChangeID.UPDATE)
- } else {
- // throw back to spark
- AlterTableAlterColumnStatement(a.tableName, a.column, a.dataType, a.nullable, a.comment, a.position)
- }
- case rename @ HoodieAlterTableRenameColumnStatement(asTable(table), col, newName) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled){
- val changes = Seq(TableChange.renameColumn(col.toArray, newName))
- Spark30AlterTableCommand(table, normalizeChanges(changes, table.schema), ColumnChangeID.UPDATE)
- } else {
- // throw back to spark
- AlterTableRenameColumnStatement(rename.tableName, rename.column, rename.newName)
- }
- case drop @ HoodieAlterTableDropColumnsStatement(asTable(table), cols) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled) {
- val changes = cols.map(col => TableChange.deleteColumn(col.toArray))
- Spark30AlterTableCommand(table, normalizeChanges(changes, table.schema), ColumnChangeID.DELETE)
- } else {
- // throw back to spark
- AlterTableDropColumnsStatement(drop.tableName, drop.columnsToDrop)
- }
- case set @ HoodieAlterTableSetPropertiesStatement(asTable(table), props) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled) {
- val changes = props.map { case (key, value) =>
- TableChange.setProperty(key, value)
- }.toSeq
- Spark30AlterTableCommand(table, normalizeChanges(changes, table.schema), ColumnChangeID.PROPERTY_CHANGE)
- } else {
- // throw back to spark
- AlterTableSetPropertiesStatement(set.tableName, set.properties)
- }
- case unset @ HoodieAlterTableUnsetPropertiesStatement(asTable(table), keys, _) =>
- if (isHoodieTable(table) && schemaEvolutionEnabled) {
- val changes = keys.map(key => TableChange.removeProperty(key))
- Spark30AlterTableCommand(table, normalizeChanges(changes, table.schema), ColumnChangeID.PROPERTY_CHANGE)
- } else {
- // throw back to spark
- AlterTableUnsetPropertiesStatement(unset.tableName, unset.propertyKeys, unset.ifExists)
- }
- }
-
- private def schemaEvolutionEnabled(): Boolean =
- sparkSession.sessionState.conf.getConfString(HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key,
- HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.defaultValue.toString).toBoolean
-
- private def isHoodieTable(table: CatalogTable): Boolean = table.provider.map(_.toLowerCase(Locale.ROOT)).orNull == "hudi"
-
- def normalizeChanges(changes: Seq[TableChange], schema: StructType): Seq[TableChange] = {
- val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
- changes.flatMap {
- case add: AddColumn =>
- def addColumn(parentSchema: StructType, parentName: String, normalizedParentName: Seq[String]): TableChange = {
- val fieldsAdded = colsToAdd.getOrElse(normalizedParentName, Nil)
- val pos = findColumnPosition(add.position(), parentName, parentSchema, fieldsAdded)
- val field = add.fieldNames().last
- colsToAdd(normalizedParentName) = fieldsAdded :+ field
- TableChange.addColumn(
- (normalizedParentName :+ field).toArray,
- add.dataType(),
- add.isNullable,
- add.comment,
- pos)
- }
- val parent = add.fieldNames().init
- if (parent.nonEmpty) {
- // Adding a nested field, need to normalize the parent column and position
- val target = schema.findNestedField(parent, includeCollections = true, SQLConf.get.resolver)
- if (target.isEmpty) {
- // Leave unresolved. Throws error in CheckAnalysis
- Some(add)
- } else {
- val (normalizedName, sf) = target.get
- sf.dataType match {
- case struct: StructType =>
- Some(addColumn(struct, parent.quoted, normalizedName :+ sf.name))
- case other =>
- Some(add)
- }
- }
- } else {
- // Adding to the root. Just need to normalize position
- Some(addColumn(schema, "root", Nil))
- }
-
- case typeChange: UpdateColumnType =>
- // Hive style syntax provides the column type, even if it may not have changed
- val fieldOpt = schema.findNestedField(
- typeChange.fieldNames(), includeCollections = true, SQLConf.get.resolver)
-
- if (fieldOpt.isEmpty) {
- // We couldn't resolve the field. Leave it to CheckAnalysis
- Some(typeChange)
- } else {
- val (fieldNames, field) = fieldOpt.get
- if (field.dataType == typeChange.newDataType()) {
- // The user didn't want the field to change, so remove this change
- None
- } else {
- Some(TableChange.updateColumnType(
- (fieldNames :+ field.name).toArray, typeChange.newDataType()))
- }
- }
- case n: UpdateColumnNullability =>
- // Need to resolve column
- resolveFieldNames(
- schema,
- n.fieldNames(),
- TableChange.updateColumnNullability(_, n.nullable())).orElse(Some(n))
-
- case position: UpdateColumnPosition =>
- position.position() match {
- case after: After =>
- // Need to resolve column as well as position reference
- val fieldOpt = schema.findNestedField(
- position.fieldNames(), includeCollections = true, SQLConf.get.resolver)
-
- if (fieldOpt.isEmpty) {
- Some(position)
- } else {
- val (normalizedPath, field) = fieldOpt.get
- val targetCol = schema.findNestedField(
- normalizedPath :+ after.column(), includeCollections = true, SQLConf.get.resolver)
- if (targetCol.isEmpty) {
- // Leave unchanged to CheckAnalysis
- Some(position)
- } else {
- Some(TableChange.updateColumnPosition(
- (normalizedPath :+ field.name).toArray,
- ColumnPosition.after(targetCol.get._2.name)))
- }
- }
- case _ =>
- // Need to resolve column
- resolveFieldNames(
- schema,
- position.fieldNames(),
- TableChange.updateColumnPosition(_, position.position())).orElse(Some(position))
- }
-
- case comment: UpdateColumnComment =>
- resolveFieldNames(
- schema,
- comment.fieldNames(),
- TableChange.updateColumnComment(_, comment.newComment())).orElse(Some(comment))
-
- case rename: RenameColumn =>
- resolveFieldNames(
- schema,
- rename.fieldNames(),
- TableChange.renameColumn(_, rename.newName())).orElse(Some(rename))
-
- case delete: DeleteColumn =>
- resolveFieldNames(schema, delete.fieldNames(), TableChange.deleteColumn)
- .orElse(Some(delete))
-
- case column: ColumnChange =>
- // This is informational for future developers
- throw new UnsupportedOperationException(
- "Please add an implementation for a column change here")
- case other => Some(other)
- }
- }
-
- /**
- * Returns the table change if the field can be resolved, returns None if the column is not
- * found. An error will be thrown in CheckAnalysis for columns that can't be resolved.
- */
- private def resolveFieldNames(
- schema: StructType,
- fieldNames: Array[String],
- copy: Array[String] => TableChange): Option[TableChange] = {
- val fieldOpt = schema.findNestedField(
- fieldNames, includeCollections = true, SQLConf.get.resolver)
- fieldOpt.map { case (path, field) => copy((path :+ field.name).toArray) }
- }
-
- private def findColumnPosition(
- position: ColumnPosition,
- parentName: String,
- struct: StructType,
- fieldsAdded: Seq[String]): ColumnPosition = {
- position match {
- case null => null
- case after: After =>
- (struct.fieldNames ++ fieldsAdded).find(n => SQLConf.get.resolver(n, after.column())) match {
- case Some(colName) =>
- ColumnPosition.after(colName)
- case None =>
- throw new AnalysisException("Couldn't find the reference column for " +
- s"$after at $parentName")
- }
- case other => other
- }
- }
-
- object asTable {
- def unapply(parts: Seq[String]): Option[CatalogTable] = {
- val identifier = parts match {
- case Seq(tblName) => TableIdentifier(tblName)
- case Seq(dbName, tblName) => TableIdentifier(tblName, Some(dbName))
- case _ =>
- throw new AnalysisException(
- s"${parts} is not a valid TableIdentifier as it has more than 2 name parts.")
- }
- Some(sparkSession.sessionState.catalog.getTableMetadata(identifier))
- }
- }
-
- private def failNullType(dt: DataType): Unit = {
- def containsNullType(dt: DataType): Boolean = dt match {
- case ArrayType(et, _) => containsNullType(et)
- case MapType(kt, vt, _) => containsNullType(kt) || containsNullType(vt)
- case StructType(fields) => fields.exists(f => containsNullType(f.dataType))
- case _ => dt.isInstanceOf[NullType]
- }
-
- if (containsNullType(dt)) {
- throw new AnalysisException(
- s"Cannot create tables with ${NullType.simpleString} type.")
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark30AlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark30AlterTableCommand.scala
deleted file mode 100644
index 177bd295e03..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark30AlterTableCommand.scala
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hudi.command
-
-import org.apache.hudi.avro.AvroSchemaUtils.getAvroRecordQualifiedName
-import org.apache.hudi.client.utils.SparkInternalSchemaConverter
-import org.apache.hudi.common.model.{HoodieCommitMetadata, WriteOperationType}
-import org.apache.hudi.common.table.timeline.HoodieInstant
-import org.apache.hudi.common.table.timeline.HoodieInstant.State
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.common.util.{CommitUtils, Option}
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
-import org.apache.hudi.internal.schema.InternalSchema
-import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID
-import org.apache.hudi.internal.schema.action.TableChanges
-import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
-import org.apache.hudi.internal.schema.io.FileBasedInternalSchemaStorageManager
-import org.apache.hudi.internal.schema.utils.{SchemaChangeUtils, SerDeHelper}
-import org.apache.hudi.table.HoodieSparkTable
-import org.apache.hudi.{DataSourceUtils, HoodieWriterUtils}
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType}
-import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, DeleteColumn, RemoveProperty, SetProperty}
-import org.apache.spark.sql.connector.catalog.{TableCatalog, TableChange}
-import org.apache.spark.sql.execution.command.RunnableCommand
-import org.apache.spark.sql.hudi.HoodieOptionConfig
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.{Row, SparkSession}
-
-import java.net.URI
-import java.nio.charset.StandardCharsets
-import java.util
-import java.util.concurrent.atomic.AtomicInteger
-import scala.collection.JavaConverters._
-import scala.util.control.NonFatal
-
-// TODO: we should remove this file when we support datasourceV2 for hoodie on spark3.0.x
-case class Spark30AlterTableCommand(table: CatalogTable, changes: Seq[TableChange], changeType: ColumnChangeID) extends RunnableCommand with Logging {
- override def run(sparkSession: SparkSession): Seq[Row] = {
- changeType match {
- case ColumnChangeID.ADD => applyAddAction(sparkSession)
- case ColumnChangeID.DELETE => applyDeleteAction(sparkSession)
- case ColumnChangeID.UPDATE => applyUpdateAction(sparkSession)
- case ColumnChangeID.PROPERTY_CHANGE if (changes.filter(_.isInstanceOf[SetProperty]).size == changes.size) =>
- applyPropertySet(sparkSession)
- case ColumnChangeID.PROPERTY_CHANGE if (changes.filter(_.isInstanceOf[RemoveProperty]).size == changes.size) =>
- applyPropertyUnset(sparkSession)
- case other => throw new RuntimeException(s"find unsupported alter command type: ${other}")
- }
- Seq.empty[Row]
- }
-
- def applyAddAction(sparkSession: SparkSession): Unit = {
- val (oldSchema, historySchema) = getInternalSchemaAndHistorySchemaStr(sparkSession)
- val addChange = TableChanges.ColumnAddChange.get(oldSchema)
- changes.map(_.asInstanceOf[AddColumn]).foreach { addColumn =>
- val names = addColumn.fieldNames()
- val parentName = Spark30AlterTableCommand.getParentName(names)
- // add col change
- val colType = SparkInternalSchemaConverter.buildTypeFromStructType(addColumn.dataType(), true, new AtomicInteger(0))
- addChange.addColumns(parentName, names.last, colType, addColumn.comment())
- // add position change
- addColumn.position() match {
- case after: TableChange.After =>
- addChange.addPositionChange(names.mkString("."),
- if (parentName.isEmpty) after.column() else parentName + "." + after.column(), "after")
- case _: TableChange.First =>
- addChange.addPositionChange(names.mkString("."), "", "first")
- case _ =>
- }
- }
- val newSchema = SchemaChangeUtils.applyTableChanges2Schema(oldSchema, addChange)
- val verifiedHistorySchema = if (historySchema == null || historySchema.isEmpty) {
- SerDeHelper.inheritSchemas(oldSchema, "")
- } else {
- historySchema
- }
- Spark30AlterTableCommand.commitWithSchema(newSchema, verifiedHistorySchema, table, sparkSession)
- logInfo("column add finished")
- }
-
- def applyDeleteAction(sparkSession: SparkSession): Unit = {
- val (oldSchema, historySchema) = getInternalSchemaAndHistorySchemaStr(sparkSession)
- val deleteChange = TableChanges.ColumnDeleteChange.get(oldSchema)
- changes.map(_.asInstanceOf[DeleteColumn]).foreach { c =>
- val originalColName = c.fieldNames().mkString(".");
- Spark30AlterTableCommand.checkSchemaChange(Seq(originalColName), table)
- deleteChange.deleteColumn(originalColName)
- }
- val newSchema = SchemaChangeUtils.applyTableChanges2Schema(oldSchema, deleteChange)
- // delete action should not change the getMaxColumnId field.
- newSchema.setMaxColumnId(oldSchema.getMaxColumnId)
- val verifiedHistorySchema = if (historySchema == null || historySchema.isEmpty) {
- SerDeHelper.inheritSchemas(oldSchema, "")
- } else {
- historySchema
- }
- Spark30AlterTableCommand.commitWithSchema(newSchema, verifiedHistorySchema, table, sparkSession)
- logInfo("column delete finished")
- }
-
- def applyUpdateAction(sparkSession: SparkSession): Unit = {
- val (oldSchema, historySchema) = getInternalSchemaAndHistorySchemaStr(sparkSession)
- val updateChange = TableChanges.ColumnUpdateChange.get(oldSchema)
- changes.foreach { change =>
- change match {
- case updateType: TableChange.UpdateColumnType =>
- val newType = SparkInternalSchemaConverter.buildTypeFromStructType(updateType.newDataType(), true, new AtomicInteger(0))
- updateChange.updateColumnType(updateType.fieldNames().mkString("."), newType)
- case updateComment: TableChange.UpdateColumnComment =>
- updateChange.updateColumnComment(updateComment.fieldNames().mkString("."), updateComment.newComment())
- case updateName: TableChange.RenameColumn =>
- val originalColName = updateName.fieldNames().mkString(".")
- Spark30AlterTableCommand.checkSchemaChange(Seq(originalColName), table)
- updateChange.renameColumn(originalColName, updateName.newName())
- case updateNullAbility: TableChange.UpdateColumnNullability =>
- updateChange.updateColumnNullability(updateNullAbility.fieldNames().mkString("."), updateNullAbility.nullable())
- case updatePosition: TableChange.UpdateColumnPosition =>
- val names = updatePosition.fieldNames()
- val parentName = Spark30AlterTableCommand.getParentName(names)
- updatePosition.position() match {
- case after: TableChange.After =>
- updateChange.addPositionChange(names.mkString("."),
- if (parentName.isEmpty) after.column() else parentName + "." + after.column(), "after")
- case _: TableChange.First =>
- updateChange.addPositionChange(names.mkString("."), "", "first")
- case _ =>
- }
- }
- }
- val newSchema = SchemaChangeUtils.applyTableChanges2Schema(oldSchema, updateChange)
- val verifiedHistorySchema = if (historySchema == null || historySchema.isEmpty) {
- SerDeHelper.inheritSchemas(oldSchema, "")
- } else {
- historySchema
- }
- Spark30AlterTableCommand.commitWithSchema(newSchema, verifiedHistorySchema, table, sparkSession)
- logInfo("column update finished")
- }
-
- // to do support unset default value to columns, and apply them to internalSchema
- def applyPropertyUnset(sparkSession: SparkSession): Unit = {
- val catalog = sparkSession.sessionState.catalog
- val propKeys = changes.map(_.asInstanceOf[RemoveProperty]).map(_.property())
- // ignore NonExist unset
- propKeys.foreach { k =>
- if (!table.properties.contains(k) && k != TableCatalog.PROP_COMMENT) {
- logWarning(s"find non exist unset property: ${k} , ignore it")
- }
- }
- val tableComment = if (propKeys.contains(TableCatalog.PROP_COMMENT)) None else table.comment
- val newProperties = table.properties.filter { case (k, _) => !propKeys.contains(k) }
- val newTable = table.copy(properties = newProperties, comment = tableComment)
- catalog.alterTable(newTable)
- logInfo("table properties change finished")
- }
-
- // to do support set default value to columns, and apply them to internalSchema
- def applyPropertySet(sparkSession: SparkSession): Unit = {
- val catalog = sparkSession.sessionState.catalog
- val properties = changes.map(_.asInstanceOf[SetProperty]).map(f => f.property -> f.value).toMap
- // This overrides old properties and update the comment parameter of CatalogTable
- // with the newly added/modified comment since CatalogTable also holds comment as its
- // direct property.
- val newTable = table.copy(
- properties = table.properties ++ properties,
- comment = properties.get(TableCatalog.PROP_COMMENT).orElse(table.comment))
- catalog.alterTable(newTable)
- logInfo("table properties change finished")
- }
-
- def getInternalSchemaAndHistorySchemaStr(sparkSession: SparkSession): (InternalSchema, String) = {
- val path = Spark30AlterTableCommand.getTableLocation(table, sparkSession)
- val storageConf = HadoopFSUtils.getStorageConf(sparkSession.sessionState.newHadoopConf())
- val metaClient = HoodieTableMetaClient.builder().setBasePath(path)
- .setConf(storageConf).build()
- val schemaUtil = new TableSchemaResolver(metaClient)
-
- val schema = schemaUtil.getTableInternalSchemaFromCommitMetadata().orElse {
- AvroInternalSchemaConverter.convert(schemaUtil.getTableAvroSchema)
- }
-
- val historySchemaStr = schemaUtil.getTableHistorySchemaStrFromCommitMetadata.orElse("")
- (schema, historySchemaStr)
- }
-}
-
-object Spark30AlterTableCommand extends Logging {
-
- /**
- * Generate an commit with new schema to change the table's schema.
- * @param internalSchema new schema after change
- * @param historySchemaStr history schemas
- * @param table The hoodie table.
- * @param sparkSession The spark session.
- */
- def commitWithSchema(internalSchema: InternalSchema, historySchemaStr: String, table: CatalogTable, sparkSession: SparkSession): Unit = {
- val schema = AvroInternalSchemaConverter.convert(internalSchema, getAvroRecordQualifiedName(table.identifier.table))
- val path = getTableLocation(table, sparkSession)
-
- val jsc = new JavaSparkContext(sparkSession.sparkContext)
- val client = DataSourceUtils.createHoodieClient(jsc, schema.toString,
- path, table.identifier.table, HoodieWriterUtils.parametersWithWriteDefaults(
- HoodieOptionConfig.mapSqlOptionsToDataSourceWriteConfigs(table.storage.properties ++ table.properties) ++
- sparkSession.sqlContext.conf.getAllConfs).asJava)
-
- val storageConf = HadoopFSUtils.getStorageConf(sparkSession.sessionState.newHadoopConf())
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(path)
- .setConf(storageConf)
- .setTimeGeneratorConfig(client.getConfig.getTimeGeneratorConfig)
- .build()
-
- val commitActionType = CommitUtils.getCommitActionType(WriteOperationType.ALTER_SCHEMA, metaClient.getTableType)
- val instantTime = client.createNewInstantTime()
- client.startCommitWithTime(instantTime, commitActionType)
- client.setOperationType(WriteOperationType.ALTER_SCHEMA)
-
- val hoodieTable = HoodieSparkTable.create(client.getConfig, client.getEngineContext)
- val timeLine = hoodieTable.getActiveTimeline
- val requested = new HoodieInstant(State.REQUESTED, commitActionType, instantTime)
- val metadata = new HoodieCommitMetadata
- metadata.setOperationType(WriteOperationType.ALTER_SCHEMA)
- timeLine.transitionRequestedToInflight(requested, Option.of(metadata.toJsonString.getBytes(StandardCharsets.UTF_8)))
- val extraMeta = new util.HashMap[String, String]()
- extraMeta.put(SerDeHelper.LATEST_SCHEMA, SerDeHelper.toJson(internalSchema.setSchemaId(instantTime.toLong)))
- val schemaManager = new FileBasedInternalSchemaStorageManager(metaClient)
- schemaManager.persistHistorySchemaStr(instantTime, SerDeHelper.inheritSchemas(internalSchema, historySchemaStr))
- client.commit(instantTime, jsc.emptyRDD, Option.of(extraMeta))
- val existRoTable = sparkSession.catalog.tableExists(table.identifier.unquotedString + "_ro")
- val existRtTable = sparkSession.catalog.tableExists(table.identifier.unquotedString + "_rt")
- try {
- sparkSession.catalog.refreshTable(table.identifier.unquotedString)
- // try to refresh ro/rt table
- if (existRoTable) sparkSession.catalog.refreshTable(table.identifier.unquotedString + "_ro")
- if (existRoTable) sparkSession.catalog.refreshTable(table.identifier.unquotedString + "_rt")
- } catch {
- case NonFatal(e) =>
- log.error(s"Exception when attempting to refresh table ${table.identifier.quotedString}", e)
- }
- // try to sync to hive
- // drop partition field before call alter table
- val fullSparkSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(internalSchema)
- val dataSparkSchema = new StructType(fullSparkSchema.fields.filter(p => !table.partitionColumnNames.exists(f => sparkSession.sessionState.conf.resolver(f, p.name))))
- alterTableDataSchema(sparkSession, table.identifier.database.getOrElse("default"), table.identifier.table, dataSparkSchema)
- if (existRoTable) alterTableDataSchema(sparkSession, table.identifier.database.getOrElse("default"), table.identifier.table + "_ro", dataSparkSchema)
- if (existRtTable) alterTableDataSchema(sparkSession, table.identifier.database.getOrElse("default"), table.identifier.table + "_rt", dataSparkSchema)
- }
-
- def alterTableDataSchema(sparkSession: SparkSession, db: String, tableName: String, dataSparkSchema: StructType): Unit = {
- sparkSession.sessionState.catalog
- .externalCatalog
- .alterTableDataSchema(db, tableName, dataSparkSchema)
- }
-
- def getTableLocation(table: CatalogTable, sparkSession: SparkSession): String = {
- val uri = if (table.tableType == CatalogTableType.MANAGED) {
- Some(sparkSession.sessionState.catalog.defaultTablePath(table.identifier))
- } else {
- table.storage.locationUri
- }
- val conf = sparkSession.sessionState.newHadoopConf()
- uri.map(makePathQualified(_, conf))
- .map(removePlaceHolder)
- .getOrElse(throw new IllegalArgumentException(s"Missing location for ${table.identifier}"))
- }
-
- private def removePlaceHolder(path: String): String = {
- if (path == null || path.length == 0) {
- path
- } else if (path.endsWith("-__PLACEHOLDER__")) {
- path.substring(0, path.length() - 16)
- } else {
- path
- }
- }
-
- def makePathQualified(path: URI, hadoopConf: Configuration): String = {
- val hadoopPath = new Path(path)
- val fs = hadoopPath.getFileSystem(hadoopConf)
- fs.makeQualified(hadoopPath).toUri.toString
- }
-
- def getParentName(names: Array[String]): String = {
- if (names.size > 1) {
- names.dropRight(1).mkString(".")
- } else ""
- }
-
- def checkSchemaChange(colNames: Seq[String], catalogTable: CatalogTable): Unit = {
- val primaryKeys = catalogTable.storage.properties.getOrElse("primaryKey", catalogTable.properties.getOrElse("primaryKey", "keyid")).split(",").map(_.trim)
- val preCombineKey = Seq(catalogTable.storage.properties.getOrElse("preCombineField", catalogTable.properties.getOrElse("preCombineField", "ts"))).map(_.trim)
- val partitionKey = catalogTable.partitionColumnNames.map(_.trim)
- val checkNames = primaryKeys ++ preCombineKey ++ partitionKey
- colNames.foreach { col =>
- if (checkNames.contains(col)) {
- throw new UnsupportedOperationException("cannot support apply changes for primaryKey/CombineKey/partitionKey")
- }
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark30SqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark30SqlAstBuilder.scala
deleted file mode 100644
index c9dad3d9a2e..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark30SqlAstBuilder.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.parser
-
-import org.apache.spark.sql.catalyst.parser.ParserUtils.withOrigin
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.execution.SparkSqlAstBuilder
-import org.apache.spark.sql.internal.SQLConf
-
-// TODO: we should remove this file when we support datasourceV2 for hoodie on spark3.0.x
-class HoodieSpark30SqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
-
- /**
- * Parse a [[AlterTableAlterColumnStatement]] command to alter a column's property.
- *
- * For example:
- * {{{
- * ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint
- * ALTER TABLE table1 ALTER COLUMN a.b.c SET NOT NULL
- * ALTER TABLE table1 ALTER COLUMN a.b.c DROP NOT NULL
- * ALTER TABLE table1 ALTER COLUMN a.b.c COMMENT 'new comment'
- * ALTER TABLE table1 ALTER COLUMN a.b.c FIRST
- * ALTER TABLE table1 ALTER COLUMN a.b.c AFTER x
- * }}}
- */
- override def visitAlterTableAlterColumn(ctx: AlterTableAlterColumnContext): LogicalPlan = withOrigin(ctx) {
- val alter = super.visitAlterTableAlterColumn(ctx).asInstanceOf[AlterTableAlterColumnStatement]
- HoodieAlterTableAlterColumnStatement(alter.tableName, alter.column, alter.dataType, alter.nullable, alter.comment, alter.position)
- }
-
- /**
- * Parse a [[org.apache.spark.sql.catalyst.plans.logical.AlterTableAddColumnsStatement]] command.
- *
- * For example:
- * {{{
- * ALTER TABLE table1
- * ADD COLUMNS (col_name data_type [COMMENT col_comment], ...);
- * }}}
- */
- override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = withOrigin(ctx) {
- val add = super.visitAddTableColumns(ctx).asInstanceOf[AlterTableAddColumnsStatement]
- HoodieAlterTableAddColumnsStatement(add.tableName, add.columnsToAdd)
- }
-
- /**
- * Parse a [[org.apache.spark.sql.catalyst.plans.logical.AlterTableRenameColumnStatement]] command.
- *
- * For example:
- * {{{
- * ALTER TABLE table1 RENAME COLUMN a.b.c TO x
- * }}}
- */
- override def visitRenameTableColumn(
- ctx: RenameTableColumnContext): LogicalPlan = withOrigin(ctx) {
- val rename = super.visitRenameTableColumn(ctx).asInstanceOf[AlterTableRenameColumnStatement]
- HoodieAlterTableRenameColumnStatement(rename.tableName, rename.column, rename.newName)
- }
-
- /**
- * Parse a [[AlterTableDropColumnsStatement]] command.
- *
- * For example:
- * {{{
- * ALTER TABLE table1 DROP COLUMN a.b.c
- * ALTER TABLE table1 DROP COLUMNS a.b.c, x, y
- * }}}
- */
- override def visitDropTableColumns(
- ctx: DropTableColumnsContext): LogicalPlan = withOrigin(ctx) {
- val drop = super.visitDropTableColumns(ctx).asInstanceOf[AlterTableDropColumnsStatement]
- HoodieAlterTableDropColumnsStatement(drop.tableName, drop.columnsToDrop)
- }
-
- /**
- * Parse [[AlterViewSetPropertiesStatement]] or [[AlterTableSetPropertiesStatement]] commands.
- *
- * For example:
- * {{{
- * ALTER TABLE table SET TBLPROPERTIES ('table_property' = 'property_value');
- * ALTER VIEW view SET TBLPROPERTIES ('table_property' = 'property_value');
- * }}}
- */
- override def visitSetTableProperties(
- ctx: SetTablePropertiesContext): LogicalPlan = withOrigin(ctx) {
- val set = super.visitSetTableProperties(ctx)
- set match {
- case s: AlterTableSetPropertiesStatement => HoodieAlterTableSetPropertiesStatement(s.tableName, s.properties)
- case other => other
- }
- }
-
- /**
- * Parse [[AlterViewUnsetPropertiesStatement]] or [[AlterTableUnsetPropertiesStatement]] commands.
- *
- * For example:
- * {{{
- * ALTER TABLE table UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key');
- * ALTER VIEW view UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key');
- * }}}
- */
- override def visitUnsetTableProperties(
- ctx: UnsetTablePropertiesContext): LogicalPlan = withOrigin(ctx) {
- val unset = super.visitUnsetTableProperties(ctx)
- unset match {
- case us: AlterTableUnsetPropertiesStatement => HoodieAlterTableUnsetPropertiesStatement(us.tableName, us.propertyKeys, us.ifExists)
- case other => other
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_0ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_0ExtendedSqlParser.scala
deleted file mode 100644
index 52343652066..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_0ExtendedSqlParser.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.parser
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.execution.{SparkSqlAstBuilder, SparkSqlParser}
-
-// TODO: we should remove this file when we support datasourceV2 for hoodie on spark3.0.x
-class HoodieSpark3_0ExtendedSqlParser(session: SparkSession, delegate: ParserInterface) extends SparkSqlParser(session.sqlContext.conf)
- with HoodieExtendedParserInterface
- with Logging {
-
- override val astBuilder: SparkSqlAstBuilder = new HoodieSpark30SqlAstBuilder(session.sqlContext.conf)
-
- override def parseMultipartIdentifier(sqlText: String): Seq[String] = super[SparkSqlParser].parseMultipartIdentifier(sqlText)
-}
-
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
deleted file mode 100644
index e6015a65cb0..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.vectorized
-
-import org.apache.spark.sql.catalyst.InternalRow
-
-object ColumnarUtils {
-
- /**
- * Utility verifying whether provided instance of [[InternalRow]] is actually
- * an instance of [[ColumnarBatchRow]]
- *
- * NOTE: This utility is required, since in Spark <= 3.3 [[ColumnarBatchRow]] is package-private
- */
- def isColumnarBatchRow(r: InternalRow): Boolean = r.isInstanceOf[ColumnarBatchRow]
-
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java b/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
deleted file mode 100644
index ea7e6e65e7c..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.internal;
-
-import org.apache.hudi.DataSourceWriteOptions;
-import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField;
-import org.apache.hudi.common.model.HoodieWriteStat;
-import org.apache.hudi.common.table.HoodieTableConfig;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
-import org.apache.hudi.testutils.SparkDatasetTestUtils;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/**
- * Base class for TestHoodieBulkInsertDataInternalWriter.
- */
-public class HoodieBulkInsertInternalWriterTestBase extends HoodieSparkClientTestHarness {
-
- protected static final Random RANDOM = new Random();
-
- @BeforeEach
- public void setUp() throws Exception {
- initSparkContexts();
- initPath();
- initHoodieStorage();
- initTestDataGenerator();
- initMetaClient();
- initTimelineService();
- }
-
- @AfterEach
- public void tearDown() throws Exception {
- cleanupResources();
- }
-
- protected HoodieWriteConfig getWriteConfig(boolean populateMetaFields) {
- return getWriteConfig(populateMetaFields, DataSourceWriteOptions.HIVE_STYLE_PARTITIONING().defaultValue());
- }
-
- protected HoodieWriteConfig getWriteConfig(boolean populateMetaFields, String hiveStylePartitioningValue) {
- Properties properties = new Properties();
- if (!populateMetaFields) {
- properties.setProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key(), SparkDatasetTestUtils.RECORD_KEY_FIELD_NAME);
- properties.setProperty(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), SparkDatasetTestUtils.PARTITION_PATH_FIELD_NAME);
- properties.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false");
- }
- properties.setProperty(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING().key(), hiveStylePartitioningValue);
- return SparkDatasetTestUtils.getConfigBuilder(basePath, timelineServicePort).withProperties(properties).build();
- }
-
- protected void assertWriteStatuses(List<WriteStatus> writeStatuses, int batches, int size,
- Option<List<String>> fileAbsPaths, Option<List<String>> fileNames) {
- assertWriteStatuses(writeStatuses, batches, size, false, fileAbsPaths, fileNames, false);
- }
-
- protected void assertWriteStatuses(List<WriteStatus> writeStatuses, int batches, int size, boolean areRecordsSorted,
- Option<List<String>> fileAbsPaths, Option<List<String>> fileNames, boolean isHiveStylePartitioning) {
- if (areRecordsSorted) {
- assertEquals(batches, writeStatuses.size());
- } else {
- assertEquals(Math.min(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS.length, batches), writeStatuses.size());
- }
-
- Map<String, Long> sizeMap = new HashMap<>();
- if (!areRecordsSorted) {
- // <size> no of records are written per batch. Every 4th batch goes into same writeStatus. So, populating the size expected
- // per write status
- for (int i = 0; i < batches; i++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[i % 3];
- if (!sizeMap.containsKey(partitionPath)) {
- sizeMap.put(partitionPath, 0L);
- }
- sizeMap.put(partitionPath, sizeMap.get(partitionPath) + size);
- }
- }
-
- int counter = 0;
- for (WriteStatus writeStatus : writeStatuses) {
- // verify write status
- String actualPartitionPathFormat = isHiveStylePartitioning ? SparkDatasetTestUtils.PARTITION_PATH_FIELD_NAME + "=%s" : "%s";
- assertEquals(String.format(actualPartitionPathFormat, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter % 3]), writeStatus.getPartitionPath());
- if (areRecordsSorted) {
- assertEquals(writeStatus.getTotalRecords(), size);
- } else {
- assertEquals(writeStatus.getTotalRecords(), sizeMap.get(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter % 3]));
- }
- assertNull(writeStatus.getGlobalError());
- assertEquals(writeStatus.getTotalErrorRecords(), 0);
- assertEquals(writeStatus.getTotalErrorRecords(), 0);
- assertFalse(writeStatus.hasErrors());
- assertNotNull(writeStatus.getFileId());
- String fileId = writeStatus.getFileId();
- if (fileAbsPaths.isPresent()) {
- fileAbsPaths.get().add(basePath + "/" + writeStatus.getStat().getPath());
- }
- if (fileNames.isPresent()) {
- fileNames.get().add(writeStatus.getStat().getPath()
- .substring(writeStatus.getStat().getPath().lastIndexOf('/') + 1));
- }
- HoodieWriteStat writeStat = writeStatus.getStat();
- if (areRecordsSorted) {
- assertEquals(size, writeStat.getNumInserts());
- assertEquals(size, writeStat.getNumWrites());
- } else {
- assertEquals(sizeMap.get(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter % 3]), writeStat.getNumInserts());
- assertEquals(sizeMap.get(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter % 3]), writeStat.getNumWrites());
- }
- assertEquals(fileId, writeStat.getFileId());
- assertEquals(String.format(actualPartitionPathFormat, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter++ % 3]), writeStat.getPartitionPath());
- assertEquals(0, writeStat.getNumDeletes());
- assertEquals(0, writeStat.getNumUpdateWrites());
- assertEquals(0, writeStat.getTotalWriteErrors());
- }
- }
-
- protected void assertOutput(Dataset<Row> expectedRows, Dataset<Row> actualRows, String instantTime, Option<List<String>> fileNames,
- boolean populateMetaColumns) {
- if (populateMetaColumns) {
- // verify 3 meta fields that are filled in within create handle
- actualRows.collectAsList().forEach(entry -> {
- assertEquals(entry.get(HoodieMetadataField.COMMIT_TIME_METADATA_FIELD.ordinal()).toString(), instantTime);
- assertFalse(entry.isNullAt(HoodieMetadataField.FILENAME_METADATA_FIELD.ordinal()));
- if (fileNames.isPresent()) {
- assertTrue(fileNames.get().contains(entry.get(HoodieMetadataField.FILENAME_METADATA_FIELD.ordinal())));
- }
- assertFalse(entry.isNullAt(HoodieMetadataField.COMMIT_SEQNO_METADATA_FIELD.ordinal()));
- });
-
- // after trimming 2 of the meta fields, rest of the fields should match
- Dataset<Row> trimmedExpected = expectedRows.drop(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, HoodieRecord.COMMIT_TIME_METADATA_FIELD, HoodieRecord.FILENAME_METADATA_FIELD);
- Dataset<Row> trimmedActual = actualRows.drop(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, HoodieRecord.COMMIT_TIME_METADATA_FIELD, HoodieRecord.FILENAME_METADATA_FIELD);
- assertEquals(0, trimmedActual.except(trimmedExpected).count());
- } else { // operation = BULK_INSERT_APPEND_ONLY
- // all meta columns are untouched
- assertEquals(0, expectedRows.except(actualRows).count());
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java b/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
deleted file mode 100644
index 206d4931b15..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.spark3.internal;
-
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.internal.HoodieBulkInsertInternalWriterTestBase;
-import org.apache.hudi.table.HoodieSparkTable;
-import org.apache.hudi.table.HoodieTable;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Stream;
-
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.ENCODER;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.STRUCT_TYPE;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.getInternalRowWithError;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.getRandomRows;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.toInternalRows;
-import static org.junit.jupiter.api.Assertions.fail;
-
-/**
- * Unit tests {@link HoodieBulkInsertDataInternalWriter}.
- */
-public class TestHoodieBulkInsertDataInternalWriter extends
- HoodieBulkInsertInternalWriterTestBase {
-
- private static Stream<Arguments> configParams() {
- Object[][] data = new Object[][] {
- {true, true},
- {true, false},
- {false, true},
- {false, false}
- };
- return Stream.of(data).map(Arguments::of);
- }
-
- private static Stream<Arguments> bulkInsertTypeParams() {
- Object[][] data = new Object[][] {
- {true},
- {false}
- };
- return Stream.of(data).map(Arguments::of);
- }
-
- @ParameterizedTest
- @MethodSource("configParams")
- public void testDataInternalWriter(boolean sorted, boolean populateMetaFields) throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(populateMetaFields);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- // execute N rounds
- for (int i = 0; i < 2; i++) {
- String instantTime = "00" + i;
- // init writer
- HoodieBulkInsertDataInternalWriter writer = new HoodieBulkInsertDataInternalWriter(table, cfg, instantTime, RANDOM.nextInt(100000),
- RANDOM.nextLong(), STRUCT_TYPE, populateMetaFields, sorted);
-
- int size = 10 + RANDOM.nextInt(1000);
- // write N rows to partition1, N rows to partition2 and N rows to partition3 ... Each batch should create a new RowCreateHandle and a new file
- int batches = 3;
- Dataset<Row> totalInputRows = null;
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- if (totalInputRows == null) {
- totalInputRows = inputRows;
- } else {
- totalInputRows = totalInputRows.union(inputRows);
- }
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- Option<List<String>> fileAbsPaths = Option.of(new ArrayList<>());
- Option<List<String>> fileNames = Option.of(new ArrayList<>());
-
- // verify write statuses
- assertWriteStatuses(commitMetadata.getWriteStatuses(), batches, size, sorted, fileAbsPaths, fileNames, false);
-
- // verify rows
- Dataset<Row> result = sqlContext.read().parquet(fileAbsPaths.get().toArray(new String[0]));
- assertOutput(totalInputRows, result, instantTime, fileNames, populateMetaFields);
- }
- }
-
-
- /**
- * Issue some corrupted or wrong schematized InternalRow after few valid InternalRows so that global error is thrown. write batch 1 of valid records write batch2 of invalid records which is expected
- * to throw Global Error. Verify global error is set appropriately and only first batch of records are written to disk.
- */
- @Test
- public void testGlobalFailure() throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(true);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[0];
-
- String instantTime = "001";
- HoodieBulkInsertDataInternalWriter writer = new HoodieBulkInsertDataInternalWriter(table, cfg, instantTime, RANDOM.nextInt(100000),
- RANDOM.nextLong(), STRUCT_TYPE, true, false);
-
- int size = 10 + RANDOM.nextInt(100);
- int totalFailures = 5;
- // Generate first batch of valid rows
- Dataset<Row> inputRows = getRandomRows(sqlContext, size / 2, partitionPath, false);
- List<InternalRow> internalRows = toInternalRows(inputRows, ENCODER);
-
- // generate some failures rows
- for (int i = 0; i < totalFailures; i++) {
- internalRows.add(getInternalRowWithError(partitionPath));
- }
-
- // generate 2nd batch of valid rows
- Dataset<Row> inputRows2 = getRandomRows(sqlContext, size / 2, partitionPath, false);
- internalRows.addAll(toInternalRows(inputRows2, ENCODER));
-
- // issue writes
- try {
- for (InternalRow internalRow : internalRows) {
- writer.write(internalRow);
- }
- fail("Should have failed");
- } catch (Throwable e) {
- // expected
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
-
- Option<List<String>> fileAbsPaths = Option.of(new ArrayList<>());
- Option<List<String>> fileNames = Option.of(new ArrayList<>());
- // verify write statuses
- assertWriteStatuses(commitMetadata.getWriteStatuses(), 1, size / 2, fileAbsPaths, fileNames);
-
- // verify rows
- Dataset<Row> result = sqlContext.read().parquet(fileAbsPaths.get().toArray(new String[0]));
- assertOutput(inputRows, result, instantTime, fileNames, true);
- }
-
- private void writeRows(Dataset<Row> inputRows, HoodieBulkInsertDataInternalWriter writer)
- throws Exception {
- List<InternalRow> internalRows = toInternalRows(inputRows, ENCODER);
- // issue writes
- for (InternalRow internalRow : internalRows) {
- writer.write(internalRow);
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java b/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
deleted file mode 100644
index 64042f2ebbb..00000000000
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.spark3.internal;
-
-import org.apache.hudi.DataSourceWriteOptions;
-import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.internal.HoodieBulkInsertInternalWriterTestBase;
-import org.apache.hudi.table.HoodieSparkTable;
-import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.testutils.HoodieClientTestUtils;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.connector.write.DataWriter;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.ENCODER;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.STRUCT_TYPE;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.getRandomRows;
-import static org.apache.hudi.testutils.SparkDatasetTestUtils.toInternalRows;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/**
- * Unit tests {@link HoodieDataSourceInternalBatchWrite}.
- */
-public class TestHoodieDataSourceInternalBatchWrite extends
- HoodieBulkInsertInternalWriterTestBase {
-
- private static Stream<Arguments> bulkInsertTypeParams() {
- Object[][] data = new Object[][] {
- {true},
- {false}
- };
- return Stream.of(data).map(Arguments::of);
- }
-
- @ParameterizedTest
- @MethodSource("bulkInsertTypeParams")
- public void testDataSourceWriter(boolean populateMetaFields) throws Exception {
- testDataSourceWriterInternal(Collections.emptyMap(), Collections.emptyMap(), populateMetaFields);
- }
-
- private void testDataSourceWriterInternal(Map<String, String> extraMetadata, Map<String, String> expectedExtraMetadata, boolean populateMetaFields) throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(populateMetaFields);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- String instantTime = "001";
- // init writer
- HoodieDataSourceInternalBatchWrite dataSourceInternalBatchWrite =
- new HoodieDataSourceInternalBatchWrite(instantTime, cfg, STRUCT_TYPE, sqlContext.sparkSession(), storageConf, extraMetadata, populateMetaFields, false);
- DataWriter<InternalRow> writer = dataSourceInternalBatchWrite.createBatchWriterFactory(null).createWriter(0, RANDOM.nextLong());
-
- String[] partitionPaths = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
- List<String> partitionPathsAbs = new ArrayList<>();
- for (String partitionPath : partitionPaths) {
- partitionPathsAbs.add(basePath + "/" + partitionPath + "/*");
- }
-
- int size = 10 + RANDOM.nextInt(1000);
- int batches = 5;
- Dataset<Row> totalInputRows = null;
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- if (totalInputRows == null) {
- totalInputRows = inputRows;
- } else {
- totalInputRows = totalInputRows.union(inputRows);
- }
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- List<HoodieWriterCommitMessage> commitMessages = new ArrayList<>();
- commitMessages.add(commitMetadata);
- dataSourceInternalBatchWrite.commit(commitMessages.toArray(new HoodieWriterCommitMessage[0]));
-
- metaClient.reloadActiveTimeline();
- Dataset<Row> result = HoodieClientTestUtils.read(
- jsc, basePath, sqlContext, metaClient.getStorage(), partitionPathsAbs.toArray(new String[0]));
- // verify output
- assertOutput(totalInputRows, result, instantTime, Option.empty(), populateMetaFields);
- assertWriteStatuses(commitMessages.get(0).getWriteStatuses(), batches, size, Option.empty(), Option.empty());
-
- // verify extra metadata
- Option<HoodieCommitMetadata> commitMetadataOption =
- HoodieClientTestUtils.getCommitMetadataForLatestInstant(metaClient);
- assertTrue(commitMetadataOption.isPresent());
- Map<String, String> actualExtraMetadata = new HashMap<>();
- commitMetadataOption.get().getExtraMetadata().entrySet().stream().filter(entry ->
- !entry.getKey().equals(HoodieCommitMetadata.SCHEMA_KEY))
- .forEach(entry -> actualExtraMetadata.put(entry.getKey(), entry.getValue()));
- assertEquals(actualExtraMetadata, expectedExtraMetadata);
- }
-
- @Test
- public void testDataSourceWriterExtraCommitMetadata() throws Exception {
- String commitExtraMetaPrefix = "commit_extra_meta_";
- Map<String, String> extraMeta = new HashMap<>();
- extraMeta.put(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX().key(), commitExtraMetaPrefix);
- extraMeta.put(commitExtraMetaPrefix + "a", "valA");
- extraMeta.put(commitExtraMetaPrefix + "b", "valB");
- extraMeta.put("commit_extra_c", "valC"); // should not be part of commit extra metadata
-
- Map<String, String> expectedMetadata = new HashMap<>();
- expectedMetadata.putAll(extraMeta);
- expectedMetadata.remove(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX().key());
- expectedMetadata.remove("commit_extra_c");
-
- testDataSourceWriterInternal(extraMeta, expectedMetadata, true);
- }
-
- @Test
- public void testDataSourceWriterEmptyExtraCommitMetadata() throws Exception {
- String commitExtraMetaPrefix = "commit_extra_meta_";
- Map<String, String> extraMeta = new HashMap<>();
- extraMeta.put(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX().key(), commitExtraMetaPrefix);
- extraMeta.put("keyA", "valA");
- extraMeta.put("keyB", "valB");
- extraMeta.put("commit_extra_c", "valC");
- // none of the keys has commit metadata key prefix.
- testDataSourceWriterInternal(extraMeta, Collections.emptyMap(), true);
- }
-
- @ParameterizedTest
- @MethodSource("bulkInsertTypeParams")
- public void testMultipleDataSourceWrites(boolean populateMetaFields) throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(populateMetaFields);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- int partitionCounter = 0;
-
- // execute N rounds
- for (int i = 0; i < 2; i++) {
- String instantTime = "00" + i;
- // init writer
- HoodieDataSourceInternalBatchWrite dataSourceInternalBatchWrite =
- new HoodieDataSourceInternalBatchWrite(instantTime, cfg, STRUCT_TYPE, sqlContext.sparkSession(), storageConf, Collections.emptyMap(), populateMetaFields, false);
- List<HoodieWriterCommitMessage> commitMessages = new ArrayList<>();
- Dataset<Row> totalInputRows = null;
- DataWriter<InternalRow> writer = dataSourceInternalBatchWrite.createBatchWriterFactory(null).createWriter(partitionCounter++, RANDOM.nextLong());
-
- int size = 10 + RANDOM.nextInt(1000);
- int batches = 3; // one batch per partition
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- if (totalInputRows == null) {
- totalInputRows = inputRows;
- } else {
- totalInputRows = totalInputRows.union(inputRows);
- }
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- commitMessages.add(commitMetadata);
- dataSourceInternalBatchWrite.commit(commitMessages.toArray(new HoodieWriterCommitMessage[0]));
- metaClient.reloadActiveTimeline();
-
- Dataset<Row> result = HoodieClientTestUtils.readCommit(basePath, sqlContext, metaClient.getCommitTimeline(), instantTime, populateMetaFields);
-
- // verify output
- assertOutput(totalInputRows, result, instantTime, Option.empty(), populateMetaFields);
- assertWriteStatuses(commitMessages.get(0).getWriteStatuses(), batches, size, Option.empty(), Option.empty());
- }
- }
-
- // Large writes are not required to be executed w/ regular CI jobs. Takes lot of running time.
- @Disabled
- @ParameterizedTest
- @MethodSource("bulkInsertTypeParams")
- public void testLargeWrites(boolean populateMetaFields) throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(populateMetaFields);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- int partitionCounter = 0;
-
- // execute N rounds
- for (int i = 0; i < 3; i++) {
- String instantTime = "00" + i;
- // init writer
- HoodieDataSourceInternalBatchWrite dataSourceInternalBatchWrite =
- new HoodieDataSourceInternalBatchWrite(instantTime, cfg, STRUCT_TYPE, sqlContext.sparkSession(), storageConf, Collections.emptyMap(), populateMetaFields, false);
- List<HoodieWriterCommitMessage> commitMessages = new ArrayList<>();
- Dataset<Row> totalInputRows = null;
- DataWriter<InternalRow> writer = dataSourceInternalBatchWrite.createBatchWriterFactory(null).createWriter(partitionCounter++, RANDOM.nextLong());
-
- int size = 10000 + RANDOM.nextInt(10000);
- int batches = 3; // one batch per partition
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- if (totalInputRows == null) {
- totalInputRows = inputRows;
- } else {
- totalInputRows = totalInputRows.union(inputRows);
- }
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- commitMessages.add(commitMetadata);
- dataSourceInternalBatchWrite.commit(commitMessages.toArray(new HoodieWriterCommitMessage[0]));
- metaClient.reloadActiveTimeline();
-
- Dataset<Row> result = HoodieClientTestUtils.readCommit(basePath, sqlContext, metaClient.getCommitTimeline(), instantTime,
- populateMetaFields);
-
- // verify output
- assertOutput(totalInputRows, result, instantTime, Option.empty(), populateMetaFields);
- assertWriteStatuses(commitMessages.get(0).getWriteStatuses(), batches, size, Option.empty(), Option.empty());
- }
- }
-
- /**
- * Tests that DataSourceWriter.abort() will abort the written records of interest write and commit batch1 write and abort batch2 Read of entire dataset should show only records from batch1.
- * commit batch1
- * abort batch2
- * verify only records from batch1 is available to read
- */
- @ParameterizedTest
- @MethodSource("bulkInsertTypeParams")
- public void testAbort(boolean populateMetaFields) throws Exception {
- // init config and table
- HoodieWriteConfig cfg = getWriteConfig(populateMetaFields);
- HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
- String instantTime0 = "00" + 0;
- // init writer
- HoodieDataSourceInternalBatchWrite dataSourceInternalBatchWrite =
- new HoodieDataSourceInternalBatchWrite(instantTime0, cfg, STRUCT_TYPE, sqlContext.sparkSession(), storageConf, Collections.emptyMap(), populateMetaFields, false);
- DataWriter<InternalRow> writer = dataSourceInternalBatchWrite.createBatchWriterFactory(null).createWriter(0, RANDOM.nextLong());
-
- List<String> partitionPaths = Arrays.asList(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS);
- List<String> partitionPathsAbs = new ArrayList<>();
- for (String partitionPath : partitionPaths) {
- partitionPathsAbs.add(basePath + "/" + partitionPath + "/*");
- }
-
- int size = 10 + RANDOM.nextInt(100);
- int batches = 1;
- Dataset<Row> totalInputRows = null;
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- if (totalInputRows == null) {
- totalInputRows = inputRows;
- } else {
- totalInputRows = totalInputRows.union(inputRows);
- }
- }
-
- HoodieWriterCommitMessage commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- List<HoodieWriterCommitMessage> commitMessages = new ArrayList<>();
- commitMessages.add(commitMetadata);
- // commit 1st batch
- dataSourceInternalBatchWrite.commit(commitMessages.toArray(new HoodieWriterCommitMessage[0]));
- metaClient.reloadActiveTimeline();
- Dataset<Row> result = HoodieClientTestUtils.read(
- jsc, basePath, sqlContext, metaClient.getStorage(), partitionPathsAbs.toArray(new String[0]));
- // verify rows
- assertOutput(totalInputRows, result, instantTime0, Option.empty(), populateMetaFields);
- assertWriteStatuses(commitMessages.get(0).getWriteStatuses(), batches, size, Option.empty(), Option.empty());
-
- // 2nd batch. abort in the end
- String instantTime1 = "00" + 1;
- dataSourceInternalBatchWrite =
- new HoodieDataSourceInternalBatchWrite(instantTime1, cfg, STRUCT_TYPE, sqlContext.sparkSession(), storageConf,
- Collections.emptyMap(), populateMetaFields, false);
- writer = dataSourceInternalBatchWrite.createBatchWriterFactory(null).createWriter(1, RANDOM.nextLong());
-
- for (int j = 0; j < batches; j++) {
- String partitionPath = HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[j % 3];
- Dataset<Row> inputRows = getRandomRows(sqlContext, size, partitionPath, false);
- writeRows(inputRows, writer);
- }
-
- commitMetadata = (HoodieWriterCommitMessage) writer.commit();
- commitMessages = new ArrayList<>();
- commitMessages.add(commitMetadata);
- // commit 1st batch
- dataSourceInternalBatchWrite.abort(commitMessages.toArray(new HoodieWriterCommitMessage[0]));
- metaClient.reloadActiveTimeline();
- result = HoodieClientTestUtils.read(
- jsc, basePath, sqlContext, metaClient.getStorage(), partitionPathsAbs.toArray(new String[0]));
- // verify rows
- // only rows from first batch should be present
- assertOutput(totalInputRows, result, instantTime0, Option.empty(), populateMetaFields);
- }
-
- private void writeRows(Dataset<Row> inputRows, DataWriter<InternalRow> writer) throws Exception {
- List<InternalRow> internalRows = toInternalRows(inputRows, ENCODER);
- // issue writes
- for (InternalRow internalRow : internalRows) {
- writer.write(internalRow);
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
deleted file mode 100644
index 6c810acdaae..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
+++ /dev/null
@@ -1,291 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hudi-spark-datasource</artifactId>
- <groupId>org.apache.hudi</groupId>
- <version>1.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>hudi-spark3.1.x_2.12</artifactId>
- <version>1.0.0-SNAPSHOT</version>
-
- <name>hudi-spark3.1.x_2.12</name>
- <packaging>jar</packaging>
-
- <properties>
- <main.basedir>${project.parent.parent.basedir}</main.basedir>
- </properties>
-
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>${scala-maven-plugin.version}</version>
- <configuration>
- <args>
- <arg>-nobootcp</arg>
- <arg>-target:jvm-1.8</arg>
- </args>
- <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- </plugins>
- </pluginManagement>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/lib</outputDirectory>
- <overWriteReleases>true</overWriteReleases>
- <overWriteSnapshots>true</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- </executions>
- <configuration>
- <skip>false</skip>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>${skip.hudi-spark3.unit.tests}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark31.version}</version>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark31.version}</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${fasterxml.spark3.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-client</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark3-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Hoodie - Test -->
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-tests-common</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-client-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-client</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-hadoop-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Parquet -->
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-avro</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark31HoodieVectorizedParquetRecordReader.java b/hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark31HoodieVectorizedParquetRecordReader.java
deleted file mode 100644
index d5108b94fce..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark31HoodieVectorizedParquetRecordReader.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hudi.client.utils.SparkInternalSchemaConverter;
-import org.apache.hudi.common.util.collection.Pair;
-import org.apache.spark.memory.MemoryMode;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.execution.vectorized.OffHeapColumnVector;
-import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
-import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.StructType;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-import java.io.IOException;
-import java.time.ZoneId;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Spark31HoodieVectorizedParquetRecordReader extends VectorizedParquetRecordReader {
-
- // save the col type change info.
- private Map<Integer, Pair<DataType, DataType>> typeChangeInfos;
-
- private ColumnarBatch columnarBatch;
-
- private Map<Integer, WritableColumnVector> idToColumnVectors;
-
- private WritableColumnVector[] columnVectors;
-
- // The capacity of vectorized batch.
- private int capacity;
-
- // If true, this class returns batches instead of rows.
- private boolean returnColumnarBatch;
-
- // The memory mode of the columnarBatch.
- private final MemoryMode memoryMode;
-
- /**
- * Batch of rows that we assemble and the current index we've returned. Every time this
- * batch is used up (batchIdx == numBatched), we populated the batch.
- */
- private int batchIdx = 0;
- private int numBatched = 0;
-
- public Spark31HoodieVectorizedParquetRecordReader(
- ZoneId convertTz,
- String datetimeRebaseMode,
- String int96RebaseMode,
- boolean useOffHeap,
- int capacity,
- Map<Integer, Pair<DataType, DataType>> typeChangeInfos) {
- super(convertTz, datetimeRebaseMode, int96RebaseMode, useOffHeap, capacity);
- memoryMode = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
- this.typeChangeInfos = typeChangeInfos;
- this.capacity = capacity;
- }
-
- @Override
- public void initBatch(StructType partitionColumns, InternalRow partitionValues) {
- super.initBatch(partitionColumns, partitionValues);
- if (columnVectors == null) {
- columnVectors = new WritableColumnVector[sparkSchema.length() + partitionColumns.length()];
- }
- if (idToColumnVectors == null) {
- idToColumnVectors = new HashMap<>();
- typeChangeInfos.entrySet()
- .stream()
- .forEach(f -> {
- WritableColumnVector vector =
- memoryMode == MemoryMode.OFF_HEAP ? new OffHeapColumnVector(capacity, f.getValue().getLeft()) : new OnHeapColumnVector(capacity, f.getValue().getLeft());
- idToColumnVectors.put(f.getKey(), vector);
- });
- }
- }
-
- @Override
- public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException, UnsupportedOperationException {
- super.initialize(inputSplit, taskAttemptContext);
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- for (Map.Entry<Integer, WritableColumnVector> e : idToColumnVectors.entrySet()) {
- e.getValue().close();
- }
- idToColumnVectors = null;
- columnarBatch = null;
- columnVectors = null;
- }
-
- @Override
- public ColumnarBatch resultBatch() {
- ColumnarBatch currentColumnBatch = super.resultBatch();
- boolean changed = false;
- for (Map.Entry<Integer, Pair<DataType, DataType>> entry : typeChangeInfos.entrySet()) {
- boolean rewrite = SparkInternalSchemaConverter
- .convertColumnVectorType((WritableColumnVector) currentColumnBatch.column(entry.getKey()),
- idToColumnVectors.get(entry.getKey()), currentColumnBatch.numRows());
- if (rewrite) {
- changed = true;
- columnVectors[entry.getKey()] = idToColumnVectors.get(entry.getKey());
- }
- }
- if (changed) {
- if (columnarBatch == null) {
- // fill other vector
- for (int i = 0; i < columnVectors.length; i++) {
- if (columnVectors[i] == null) {
- columnVectors[i] = (WritableColumnVector) currentColumnBatch.column(i);
- }
- }
- columnarBatch = new ColumnarBatch(columnVectors);
- }
- columnarBatch.setNumRows(currentColumnBatch.numRows());
- return columnarBatch;
- } else {
- return currentColumnBatch;
- }
- }
-
- @Override
- public boolean nextBatch() throws IOException {
- boolean result = super.nextBatch();
- if (idToColumnVectors != null) {
- idToColumnVectors.entrySet().stream().forEach(e -> e.getValue().reset());
- }
- numBatched = resultBatch().numRows();
- batchIdx = 0;
- return result;
- }
-
- @Override
- public void enableReturningBatches() {
- returnColumnarBatch = true;
- super.enableReturningBatches();
- }
-
- @Override
- public Object getCurrentValue() {
- if (typeChangeInfos == null || typeChangeInfos.isEmpty()) {
- return super.getCurrentValue();
- }
-
- if (returnColumnarBatch) {
- return columnarBatch == null ? super.getCurrentValue() : columnarBatch;
- }
-
- return columnarBatch == null ? super.getCurrentValue() : columnarBatch.getRow(batchIdx - 1);
- }
-
- @Override
- public boolean nextKeyValue() throws IOException {
- resultBatch();
-
- if (returnColumnarBatch) {
- return nextBatch();
- }
-
- if (batchIdx >= numBatched) {
- if (!nextBatch()) {
- return false;
- }
- }
- ++batchIdx;
- return true;
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/hudi-spark-datasource/hudi-spark3.1.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
deleted file mode 100644
index faf4a1d516c..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ /dev/null
@@ -1,19 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-org.apache.hudi.Spark31DefaultSource
\ No newline at end of file
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31DefaultSource.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31DefaultSource.scala
deleted file mode 100644
index 61b15b44a67..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31DefaultSource.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi
-
-import org.apache.spark.sql.sources.DataSourceRegister
-
-class Spark31DefaultSource extends DefaultSource with DataSourceRegister {
- override def shortName(): String = "hudi"
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala
deleted file mode 100644
index c9a8f07b464..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi
-
-import org.apache.hudi.HoodieUnsafeRDD
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile}
-import org.apache.spark.sql.types.StructType
-
-class Spark31HoodieFileScanRDD(@transient private val sparkSession: SparkSession,
- read: PartitionedFile => Iterator[InternalRow],
- @transient filePartitions: Seq[FilePartition])
- extends FileScanRDD(sparkSession, read, filePartitions)
- with HoodieUnsafeRDD {
-
- override final def collect(): Array[InternalRow] = super[HoodieUnsafeRDD].collect()
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalogUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalogUtils.scala
deleted file mode 100644
index 8eaf768c8f7..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalogUtils.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.connector.expressions.{BucketTransform, NamedReference, Transform}
-
-object HoodieSpark31CatalogUtils extends HoodieSpark3CatalogUtils {
-
- override def unapplyBucketTransform(t: Transform): Option[(Int, Seq[NamedReference], Seq[NamedReference])] =
- t match {
- case BucketTransform(numBuckets, ref) => Some(numBuckets, Seq(ref), Seq.empty)
- case _ => None
- }
-
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystExpressionUtils.scala
deleted file mode 100644
index 3d32b206fd1..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystExpressionUtils.scala
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.HoodieSparkTypeUtils.isCastPreservingOrdering
-import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
-import org.apache.spark.sql.catalyst.expressions.{Add, AnsiCast, Attribute, AttributeReference, AttributeSet, BitwiseOr, Cast, DateAdd, DateDiff, DateFormatClass, DateSub, Divide, Exp, Expm1, Expression, FromUTCTimestamp, FromUnixTime, Log, Log10, Log1p, Log2, Lower, Multiply, ParseToDate, ParseToTimestamp, PredicateHelper, ShiftLeft, ShiftRight, ToUTCTimestamp, ToUnixTimestamp, Upper}
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.types.{DataType, StructType}
-object HoodieSpark31CatalystExpressionUtils extends HoodieSpark3CatalystExpressionUtils with PredicateHelper {
-
- override def getEncoder(schema: StructType): ExpressionEncoder[Row] = {
- RowEncoder.apply(schema).resolveAndBind()
- }
-
- override def normalizeExprs(exprs: Seq[Expression], attributes: Seq[Attribute]): Seq[Expression] =
- DataSourceStrategy.normalizeExprs(exprs, attributes)
-
- override def extractPredicatesWithinOutputSet(condition: Expression,
- outputSet: AttributeSet): Option[Expression] = {
- super[PredicateHelper].extractPredicatesWithinOutputSet(condition, outputSet)
- }
-
- override def matchCast(expr: Expression): Option[(Expression, DataType, Option[String])] =
- expr match {
- case Cast(child, dataType, timeZoneId) => Some((child, dataType, timeZoneId))
- case _ => None
- }
-
- override def tryMatchAttributeOrderingPreservingTransformation(expr: Expression): Option[AttributeReference] = {
- expr match {
- case OrderPreservingTransformation(attrRef) => Some(attrRef)
- case _ => None
- }
- }
-
- def canUpCast(fromType: DataType, toType: DataType): Boolean =
- Cast.canUpCast(fromType, toType)
-
- override def unapplyCastExpression(expr: Expression): Option[(Expression, DataType, Option[String], Boolean)] =
- expr match {
- case Cast(castedExpr, dataType, timeZoneId) => Some((castedExpr, dataType, timeZoneId, false))
- case AnsiCast(castedExpr, dataType, timeZoneId) => Some((castedExpr, dataType, timeZoneId, true))
- case _ => None
- }
-
- private object OrderPreservingTransformation {
- def unapply(expr: Expression): Option[AttributeReference] = {
- expr match {
- // Date/Time Expressions
- case DateFormatClass(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case DateAdd(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateSub(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateDiff(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case DateDiff(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case FromUnixTime(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case FromUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case ParseToDate(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case ParseToTimestamp(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case ToUnixTimestamp(OrderPreservingTransformation(attrRef), _, _, _) => Some(attrRef)
- case ToUTCTimestamp(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
-
- // String Expressions
- case Lower(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Upper(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case org.apache.spark.sql.catalyst.expressions.Left(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
-
- // Math Expressions
- // Binary
- case Add(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case Add(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case Multiply(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case Multiply(_, OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case Divide(OrderPreservingTransformation(attrRef), _, _) => Some(attrRef)
- case BitwiseOr(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case BitwiseOr(_, OrderPreservingTransformation(attrRef)) => Some(attrRef)
- // Unary
- case Exp(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Expm1(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log10(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log1p(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case Log2(OrderPreservingTransformation(attrRef)) => Some(attrRef)
- case ShiftLeft(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
- case ShiftRight(OrderPreservingTransformation(attrRef), _) => Some(attrRef)
-
- // Other
- case cast @ Cast(OrderPreservingTransformation(attrRef), _, _)
- if isCastPreservingOrdering(cast.child.dataType, cast.dataType) => Some(attrRef)
-
- // Identity transformation
- case attrRef: AttributeReference => Some(attrRef)
- // No match
- case _ => None
- }
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
deleted file mode 100644
index 7462d41d299..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.ResolvedTable
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema}
-import org.apache.spark.sql.catalyst.planning.ScanOperation
-import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, MergeIntoTable}
-import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
-import org.apache.spark.sql.execution.command.AlterTableRecoverPartitionsCommand
-import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat}
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-import org.apache.spark.sql.types.StructType
-
-object HoodieSpark31CatalystPlanUtils extends HoodieSpark3CatalystPlanUtils {
-
- def unapplyResolvedTable(plan: LogicalPlan): Option[(TableCatalog, Identifier, Table)] =
- plan match {
- case ResolvedTable(catalog, identifier, table) => Some((catalog, identifier, table))
- case _ => None
- }
-
- override def unapplyMergeIntoTable(plan: LogicalPlan): Option[(LogicalPlan, LogicalPlan, Expression)] = {
- plan match {
- case MergeIntoTable(targetTable, sourceTable, mergeCondition, _, _) =>
- Some((targetTable, sourceTable, mergeCondition))
- case _ => None
- }
- }
-
- override def maybeApplyForNewFileFormat(plan: LogicalPlan): LogicalPlan = {
- plan match {
- case s@ScanOperation(_, _,
- l@LogicalRelation(fs: HadoopFsRelation, _, _, _))
- if fs.fileFormat.isInstanceOf[ParquetFileFormat with HoodieFormatTrait]
- && !fs.fileFormat.asInstanceOf[ParquetFileFormat with HoodieFormatTrait].isProjected =>
- FileFormatUtilsForFileGroupReader.applyNewFileFormatChanges(s, l, fs)
- case _ => plan
- }
- }
-
- override def projectOverSchema(schema: StructType, output: AttributeSet): ProjectionOverSchema = ProjectionOverSchema(schema)
-
- override def isRepairTable(plan: LogicalPlan): Boolean = {
- plan.isInstanceOf[AlterTableRecoverPartitionsCommand]
- }
-
- override def getRepairTableChildren(plan: LogicalPlan): Option[(TableIdentifier, Boolean, Boolean, String)] = {
- plan match {
- // For Spark >= 3.2.x, AlterTableRecoverPartitionsCommand was renamed RepairTableCommand, and added two new
- // parameters: enableAddPartitions and enableDropPartitions. By setting them to true and false, can restore
- // AlterTableRecoverPartitionsCommand's behavior
- case c: AlterTableRecoverPartitionsCommand =>
- Some((c.tableName, true, false, c.cmd))
- }
- }
-
- /**
- * Managing Indexes commands are not supported for Spark3.1
- */
- override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = None
-
- override def unapplyDropIndex(plan: LogicalPlan): Option[(LogicalPlan, String, Boolean)] = None
-
- override def unapplyShowIndexes(plan: LogicalPlan): Option[(LogicalPlan, Seq[Attribute])] = None
-
- override def unapplyRefreshIndex(plan: LogicalPlan): Option[(LogicalPlan, String)] = None
-
- override def unapplyInsertIntoStatement(plan: LogicalPlan): Option[(LogicalPlan, Seq[String], Map[String, Option[String]], LogicalPlan, Boolean, Boolean)] = {
- plan match {
- case insert: InsertIntoStatement =>
- Some((insert.table, Seq.empty, insert.partitionSpec, insert.query, insert.overwrite, insert.ifPartitionNotExists))
- case _ =>
- None
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
deleted file mode 100644
index 49388f55791..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.SchemaUtils
-
-/**
- * Utils on schema for Spark 3.1.
- */
-object HoodieSpark31SchemaUtils extends HoodieSchemaUtils {
- override def checkColumnNameDuplication(columnNames: Seq[String],
- colType: String,
- caseSensitiveAnalysis: Boolean): Unit = {
- SchemaUtils.checkColumnNameDuplication(columnNames, colType, caseSensitiveAnalysis)
- }
-
- override def toAttributes(struct: StructType): Seq[Attribute] = {
- struct.toAttributes
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala
deleted file mode 100644
index 21f897afe1c..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.adapter
-
-import org.apache.avro.Schema
-import org.apache.hadoop.conf.Configuration
-import org.apache.hudi.Spark31HoodieFileScanRDD
-import org.apache.spark.sql._
-import org.apache.spark.sql.avro.{HoodieAvroDeserializer, HoodieAvroSerializer, HoodieSpark3_1AvroDeserializer, HoodieSpark3_1AvroSerializer}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.catalog.CatalogTable
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.connector.catalog.V2TableWithV1Fallback
-import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, Spark31LegacyHoodieParquetFileFormat, Spark31ParquetReader, SparkParquetReader}
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, HoodieSpark31PartitionedFileUtils, HoodieSparkPartitionedFileUtils, PartitionedFile}
-import org.apache.spark.sql.hudi.SparkAdapter
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.parser.{HoodieExtendedParserInterface, HoodieSpark3_1ExtendedSqlParser}
-import org.apache.spark.sql.types.{DataType, Metadata, MetadataBuilder, StructType}
-import org.apache.spark.sql.vectorized.ColumnarUtils
-import org.apache.spark.storage.StorageLevel
-import org.apache.spark.storage.StorageLevel._
-
-/**
- * Implementation of [[SparkAdapter]] for Spark 3.1.x
- */
-class Spark3_1Adapter extends BaseSpark3Adapter {
-
- override def resolveHoodieTable(plan: LogicalPlan): Option[CatalogTable] = {
- super.resolveHoodieTable(plan).orElse {
- EliminateSubqueryAliases(plan) match {
- // First, we need to weed out unresolved plans
- case plan if !plan.resolved => None
- // NOTE: When resolving Hudi table we allow [[Filter]]s and [[Project]]s be applied
- // on top of it
- case PhysicalOperation(_, _, DataSourceV2Relation(v2: V2TableWithV1Fallback, _, _, _, _)) if isHoodieTable(v2.v1Table) =>
- Some(v2.v1Table)
- case _ => None
- }
- }
- }
-
- override def isColumnarBatchRow(r: InternalRow): Boolean = ColumnarUtils.isColumnarBatchRow(r)
-
- def createCatalystMetadataForMetaField: Metadata =
- // NOTE: Since [[METADATA_COL_ATTR_KEY]] flag is not available in Spark 2.x,
- // we simply produce an empty [[Metadata]] instance
- new MetadataBuilder().build()
-
- override def getCatalogUtils: HoodieSpark3CatalogUtils = HoodieSpark31CatalogUtils
-
- override def getCatalystPlanUtils: HoodieCatalystPlansUtils = HoodieSpark31CatalystPlanUtils
-
- override def getCatalystExpressionUtils: HoodieCatalystExpressionUtils = HoodieSpark31CatalystExpressionUtils
-
- override def getSchemaUtils: HoodieSchemaUtils = HoodieSpark31SchemaUtils
-
- override def getSparkPartitionedFileUtils: HoodieSparkPartitionedFileUtils = HoodieSpark31PartitionedFileUtils
-
- override def createAvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean): HoodieAvroSerializer =
- new HoodieSpark3_1AvroSerializer(rootCatalystType, rootAvroType, nullable)
-
- override def createAvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType): HoodieAvroDeserializer =
- new HoodieSpark3_1AvroDeserializer(rootAvroType, rootCatalystType)
-
- override def createExtendedSparkParser(spark: SparkSession, delegate: ParserInterface): HoodieExtendedParserInterface =
- new HoodieSpark3_1ExtendedSqlParser(spark, delegate)
-
- override def createLegacyHoodieParquetFileFormat(appendPartitionValues: Boolean): Option[ParquetFileFormat] = {
- Some(new Spark31LegacyHoodieParquetFileFormat(appendPartitionValues))
- }
-
- override def createHoodieFileScanRDD(sparkSession: SparkSession,
- readFunction: PartitionedFile => Iterator[InternalRow],
- filePartitions: Seq[FilePartition],
- readDataSchema: StructType,
- metadataColumns: Seq[AttributeReference] = Seq.empty): FileScanRDD = {
- new Spark31HoodieFileScanRDD(sparkSession, readFunction, filePartitions)
- }
-
- override def extractDeleteCondition(deleteFromTable: Command): Expression = {
- deleteFromTable.asInstanceOf[DeleteFromTable].condition.getOrElse(null)
- }
-
- /**
- * Converts instance of [[StorageLevel]] to a corresponding string
- */
- override def convertStorageLevelToString(level: StorageLevel): String = level match {
- case NONE => "NONE"
- case DISK_ONLY => "DISK_ONLY"
- case DISK_ONLY_2 => "DISK_ONLY_2"
- case DISK_ONLY_3 => "DISK_ONLY_3"
- case MEMORY_ONLY => "MEMORY_ONLY"
- case MEMORY_ONLY_2 => "MEMORY_ONLY_2"
- case MEMORY_ONLY_SER => "MEMORY_ONLY_SER"
- case MEMORY_ONLY_SER_2 => "MEMORY_ONLY_SER_2"
- case MEMORY_AND_DISK => "MEMORY_AND_DISK"
- case MEMORY_AND_DISK_2 => "MEMORY_AND_DISK_2"
- case MEMORY_AND_DISK_SER => "MEMORY_AND_DISK_SER"
- case MEMORY_AND_DISK_SER_2 => "MEMORY_AND_DISK_SER_2"
- case OFF_HEAP => "OFF_HEAP"
- case _ => throw new IllegalArgumentException(s"Invalid StorageLevel: $level")
- }
-
- /**
- * Get parquet file reader
- *
- * @param vectorized true if vectorized reading is not prohibited due to schema, reading mode, etc
- * @param sqlConf the [[SQLConf]] used for the read
- * @param options passed as a param to the file format
- * @param hadoopConf some configs will be set for the hadoopConf
- * @return parquet file reader
- */
- override def createParquetFileReader(vectorized: Boolean,
- sqlConf: SQLConf,
- options: Map[String, String],
- hadoopConf: Configuration): SparkParquetReader = {
- Spark31ParquetReader.build(vectorized, sqlConf, options, hadoopConf)
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
deleted file mode 100644
index 1e3651c87ec..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ /dev/null
@@ -1,497 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Conversions.DecimalConversion
-import org.apache.avro.LogicalTypes.{TimestampMicros, TimestampMillis}
-import org.apache.avro.Schema.Type._
-import org.apache.avro.generic._
-import org.apache.avro.util.Utf8
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
-import org.apache.spark.sql.avro.AvroDeserializer.{createDateRebaseFuncInRead, createTimestampRebaseFuncInRead}
-import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
-import org.apache.spark.sql.execution.datasources.DataSourceUtils
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
-
-import java.math.BigDecimal
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * A deserializer to deserialize data in avro format to data in catalyst format.
- *
- * NOTE: This code is borrowed from Spark 3.1.2
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[sql] class AvroDeserializer(rootAvroType: Schema,
- rootCatalystType: DataType,
- datetimeRebaseMode: LegacyBehaviorPolicy.Value,
- filters: StructFilters) {
-
- def this(rootAvroType: Schema, rootCatalystType: DataType) = {
- this(
- rootAvroType,
- rootCatalystType,
- LegacyBehaviorPolicy.withName(SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ,
- LegacyBehaviorPolicy.CORRECTED.toString)),
- new NoopFilters)
- }
-
- private lazy val decimalConversions = new DecimalConversion()
-
- private val dateRebaseFunc = createDateRebaseFuncInRead(
- datetimeRebaseMode, "Avro")
-
- private val timestampRebaseFunc = createTimestampRebaseFuncInRead(
- datetimeRebaseMode, "Avro")
-
- private val converter: Any => Option[Any] = rootCatalystType match {
- // A shortcut for empty schema.
- case st: StructType if st.isEmpty =>
- (data: Any) => Some(InternalRow.empty)
-
- case st: StructType =>
- val resultRow = new SpecificInternalRow(st.map(_.dataType))
- val fieldUpdater = new RowUpdater(resultRow)
- val applyFilters = filters.skipRow(resultRow, _)
- val writer = getRecordWriter(rootAvroType, st, Nil, applyFilters)
- (data: Any) => {
- val record = data.asInstanceOf[GenericRecord]
- val skipRow = writer(fieldUpdater, record)
- if (skipRow) None else Some(resultRow)
- }
-
- case _ =>
- val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
- val fieldUpdater = new RowUpdater(tmpRow)
- val writer = newWriter(rootAvroType, rootCatalystType, Nil)
- (data: Any) => {
- writer(fieldUpdater, 0, data)
- Some(tmpRow.get(0, rootCatalystType))
- }
- }
-
- def deserialize(data: Any): Option[Any] = converter(data)
-
- /**
- * Creates a writer to write avro values to Catalyst values at the given ordinal with the given
- * updater.
- */
- private def newWriter(avroType: Schema,
- catalystType: DataType,
- path: List[String]): (CatalystDataUpdater, Int, Any) => Unit =
- (avroType.getType, catalystType) match {
- case (NULL, NullType) => (updater, ordinal, _) =>
- updater.setNullAt(ordinal)
-
- // TODO: we can avoid boxing if future version of avro provide primitive accessors.
- case (BOOLEAN, BooleanType) => (updater, ordinal, value) =>
- updater.setBoolean(ordinal, value.asInstanceOf[Boolean])
-
- case (INT, IntegerType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, value.asInstanceOf[Int])
-
- case (INT, DateType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int]))
-
- case (LONG, LongType) => (updater, ordinal, value) =>
- updater.setLong(ordinal, value.asInstanceOf[Long])
-
- case (LONG, TimestampType) => avroType.getLogicalType match {
- // For backward compatibility, if the Avro type is Long and it is not logical type
- // (the `null` case), the value is processed as timestamp type with millisecond precision.
- case null | _: TimestampMillis => (updater, ordinal, value) =>
- val millis = value.asInstanceOf[Long]
- val micros = DateTimeUtils.millisToMicros(millis)
- updater.setLong(ordinal, timestampRebaseFunc(micros))
- case _: TimestampMicros => (updater, ordinal, value) =>
- val micros = value.asInstanceOf[Long]
- updater.setLong(ordinal, timestampRebaseFunc(micros))
- case other => throw new IncompatibleSchemaException(
- s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
- }
-
- // Before we upgrade Avro to 1.8 for logical type support, spark-avro converts Long to Date.
- // For backward compatibility, we still keep this conversion.
- case (LONG, DateType) => (updater, ordinal, value) =>
- updater.setInt(ordinal, (value.asInstanceOf[Long] / MILLIS_PER_DAY).toInt)
-
- case (FLOAT, FloatType) => (updater, ordinal, value) =>
- updater.setFloat(ordinal, value.asInstanceOf[Float])
-
- case (DOUBLE, DoubleType) => (updater, ordinal, value) =>
- updater.setDouble(ordinal, value.asInstanceOf[Double])
-
- case (STRING, StringType) => (updater, ordinal, value) =>
- val str = value match {
- case s: String => UTF8String.fromString(s)
- case s: Utf8 =>
- val bytes = new Array[Byte](s.getByteLength)
- System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
- UTF8String.fromBytes(bytes)
- case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
- }
- updater.set(ordinal, str)
-
- case (ENUM, StringType) => (updater, ordinal, value) =>
- updater.set(ordinal, UTF8String.fromString(value.toString))
-
- case (FIXED, BinaryType) => (updater, ordinal, value) =>
- updater.set(ordinal, value.asInstanceOf[GenericFixed].bytes().clone())
-
- case (BYTES, BinaryType) => (updater, ordinal, value) =>
- val bytes = value match {
- case b: ByteBuffer =>
- val bytes = new Array[Byte](b.remaining)
- b.get(bytes)
- // Do not forget to reset the position
- b.rewind()
- bytes
- case b: Array[Byte] => b
- case other => throw new RuntimeException(s"$other is not a valid avro binary.")
- }
- updater.set(ordinal, bytes)
-
- case (FIXED, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
- val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d)
- val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
- updater.setDecimal(ordinal, decimal)
-
- case (BYTES, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
- val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d)
- val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
- updater.setDecimal(ordinal, decimal)
-
- case (RECORD, st: StructType) =>
- // Avro datasource doesn't accept filters with nested attributes. See SPARK-32328.
- // We can always return `false` from `applyFilters` for nested records.
- val writeRecord = getRecordWriter(avroType, st, path, applyFilters = _ => false)
- (updater, ordinal, value) =>
- val row = new SpecificInternalRow(st)
- writeRecord(new RowUpdater(row), value.asInstanceOf[GenericRecord])
- updater.set(ordinal, row)
-
- case (ARRAY, ArrayType(elementType, containsNull)) =>
- val elementWriter = newWriter(avroType.getElementType, elementType, path)
- (updater, ordinal, value) =>
- val collection = value.asInstanceOf[java.util.Collection[Any]]
- val result = createArrayData(elementType, collection.size())
- val elementUpdater = new ArrayDataUpdater(result)
-
- var i = 0
- val iter = collection.iterator()
- while (iter.hasNext) {
- val element = iter.next()
- if (element == null) {
- if (!containsNull) {
- throw new RuntimeException(s"Array value at path ${path.mkString(".")} is not " +
- "allowed to be null")
- } else {
- elementUpdater.setNullAt(i)
- }
- } else {
- elementWriter(elementUpdater, i, element)
- }
- i += 1
- }
-
- updater.set(ordinal, result)
-
- case (MAP, MapType(keyType, valueType, valueContainsNull)) if keyType == StringType =>
- val keyWriter = newWriter(SchemaBuilder.builder().stringType(), StringType, path)
- val valueWriter = newWriter(avroType.getValueType, valueType, path)
- (updater, ordinal, value) =>
- val map = value.asInstanceOf[java.util.Map[AnyRef, AnyRef]]
- val keyArray = createArrayData(keyType, map.size())
- val keyUpdater = new ArrayDataUpdater(keyArray)
- val valueArray = createArrayData(valueType, map.size())
- val valueUpdater = new ArrayDataUpdater(valueArray)
- val iter = map.entrySet().iterator()
- var i = 0
- while (iter.hasNext) {
- val entry = iter.next()
- assert(entry.getKey != null)
- keyWriter(keyUpdater, i, entry.getKey)
- if (entry.getValue == null) {
- if (!valueContainsNull) {
- throw new RuntimeException(s"Map value at path ${path.mkString(".")} is not " +
- "allowed to be null")
- } else {
- valueUpdater.setNullAt(i)
- }
- } else {
- valueWriter(valueUpdater, i, entry.getValue)
- }
- i += 1
- }
-
- // The Avro map will never have null or duplicated map keys, it's safe to create a
- // ArrayBasedMapData directly here.
- updater.set(ordinal, new ArrayBasedMapData(keyArray, valueArray))
-
- case (UNION, _) =>
- val allTypes = avroType.getTypes.asScala
- val nonNullTypes = allTypes.filter(_.getType != NULL)
- val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava)
- if (nonNullTypes.nonEmpty) {
- if (nonNullTypes.length == 1) {
- newWriter(nonNullTypes.head, catalystType, path)
- } else {
- nonNullTypes.map(_.getType).toSeq match {
- case Seq(a, b) if Set(a, b) == Set(INT, LONG) && catalystType == LongType =>
- (updater, ordinal, value) =>
- value match {
- case null => updater.setNullAt(ordinal)
- case l: java.lang.Long => updater.setLong(ordinal, l)
- case i: java.lang.Integer => updater.setLong(ordinal, i.longValue())
- }
-
- case Seq(a, b) if Set(a, b) == Set(FLOAT, DOUBLE) && catalystType == DoubleType =>
- (updater, ordinal, value) =>
- value match {
- case null => updater.setNullAt(ordinal)
- case d: java.lang.Double => updater.setDouble(ordinal, d)
- case f: java.lang.Float => updater.setDouble(ordinal, f.doubleValue())
- }
-
- case _ =>
- catalystType match {
- case st: StructType if st.length == nonNullTypes.size =>
- val fieldWriters = nonNullTypes.zip(st.fields).map {
- case (schema, field) => newWriter(schema, field.dataType, path :+ field.name)
- }.toArray
- (updater, ordinal, value) => {
- val row = new SpecificInternalRow(st)
- val fieldUpdater = new RowUpdater(row)
- val i = GenericData.get().resolveUnion(nonNullAvroType, value)
- fieldWriters(i)(fieldUpdater, i, value)
- updater.set(ordinal, row)
- }
-
- case _ =>
- throw new IncompatibleSchemaException(
- s"Cannot convert Avro to catalyst because schema at path " +
- s"${path.mkString(".")} is not compatible " +
- s"(avroType = $avroType, sqlType = $catalystType).\n" +
- s"Source Avro schema: $rootAvroType.\n" +
- s"Target Catalyst type: $rootCatalystType")
- }
- }
- }
- } else {
- (updater, ordinal, value) => updater.setNullAt(ordinal)
- }
-
- case _ =>
- throw new IncompatibleSchemaException(
- s"Cannot convert Avro to catalyst because schema at path ${path.mkString(".")} " +
- s"is not compatible (avroType = $avroType, sqlType = $catalystType).\n" +
- s"Source Avro schema: $rootAvroType.\n" +
- s"Target Catalyst type: $rootCatalystType")
- }
-
- // TODO: move the following method in Decimal object on creating Decimal from BigDecimal?
- private def createDecimal(decimal: BigDecimal, precision: Int, scale: Int): Decimal = {
- if (precision <= Decimal.MAX_LONG_DIGITS) {
- // Constructs a `Decimal` with an unscaled `Long` value if possible.
- Decimal(decimal.unscaledValue().longValue(), precision, scale)
- } else {
- // Otherwise, resorts to an unscaled `BigInteger` instead.
- Decimal(decimal, precision, scale)
- }
- }
-
- private def getRecordWriter(avroType: Schema,
- sqlType: StructType,
- path: List[String],
- applyFilters: Int => Boolean): (CatalystDataUpdater, GenericRecord) => Boolean = {
- val validFieldIndexes = ArrayBuffer.empty[Int]
- val fieldWriters = ArrayBuffer.empty[(CatalystDataUpdater, Any) => Unit]
-
- val avroSchemaHelper = new AvroUtils.AvroSchemaHelper(avroType)
- val length = sqlType.length
- var i = 0
- while (i < length) {
- val sqlField = sqlType.fields(i)
- avroSchemaHelper.getFieldByName(sqlField.name) match {
- case Some(avroField) =>
- validFieldIndexes += avroField.pos()
-
- val baseWriter = newWriter(avroField.schema(), sqlField.dataType, path :+ sqlField.name)
- val ordinal = i
- val fieldWriter = (fieldUpdater: CatalystDataUpdater, value: Any) => {
- if (value == null) {
- fieldUpdater.setNullAt(ordinal)
- } else {
- baseWriter(fieldUpdater, ordinal, value)
- }
- }
- fieldWriters += fieldWriter
- case None if !sqlField.nullable =>
- val fieldStr = s"${path.mkString(".")}.${sqlField.name}"
- throw new IncompatibleSchemaException(
- s"""
- |Cannot find non-nullable field $fieldStr in Avro schema.
- |Source Avro schema: $rootAvroType.
- |Target Catalyst type: $rootCatalystType.
- """.stripMargin)
- case _ => // nothing to do
- }
- i += 1
- }
-
- (fieldUpdater, record) => {
- var i = 0
- var skipRow = false
- while (i < validFieldIndexes.length && !skipRow) {
- fieldWriters(i)(fieldUpdater, record.get(validFieldIndexes(i)))
- skipRow = applyFilters(i)
- i += 1
- }
- skipRow
- }
- }
-
- private def createArrayData(elementType: DataType, length: Int): ArrayData = elementType match {
- case BooleanType => UnsafeArrayData.fromPrimitiveArray(new Array[Boolean](length))
- case ByteType => UnsafeArrayData.fromPrimitiveArray(new Array[Byte](length))
- case ShortType => UnsafeArrayData.fromPrimitiveArray(new Array[Short](length))
- case IntegerType => UnsafeArrayData.fromPrimitiveArray(new Array[Int](length))
- case LongType => UnsafeArrayData.fromPrimitiveArray(new Array[Long](length))
- case FloatType => UnsafeArrayData.fromPrimitiveArray(new Array[Float](length))
- case DoubleType => UnsafeArrayData.fromPrimitiveArray(new Array[Double](length))
- case _ => new GenericArrayData(new Array[Any](length))
- }
-
- /**
- * A base interface for updating values inside catalyst data structure like `InternalRow` and
- * `ArrayData`.
- */
- sealed trait CatalystDataUpdater {
- def set(ordinal: Int, value: Any): Unit
-
- def setNullAt(ordinal: Int): Unit = set(ordinal, null)
-
- def setBoolean(ordinal: Int, value: Boolean): Unit = set(ordinal, value)
-
- def setByte(ordinal: Int, value: Byte): Unit = set(ordinal, value)
-
- def setShort(ordinal: Int, value: Short): Unit = set(ordinal, value)
-
- def setInt(ordinal: Int, value: Int): Unit = set(ordinal, value)
-
- def setLong(ordinal: Int, value: Long): Unit = set(ordinal, value)
-
- def setDouble(ordinal: Int, value: Double): Unit = set(ordinal, value)
-
- def setFloat(ordinal: Int, value: Float): Unit = set(ordinal, value)
-
- def setDecimal(ordinal: Int, value: Decimal): Unit = set(ordinal, value)
- }
-
- final class RowUpdater(row: InternalRow) extends CatalystDataUpdater {
- override def set(ordinal: Int, value: Any): Unit = row.update(ordinal, value)
-
- override def setNullAt(ordinal: Int): Unit = row.setNullAt(ordinal)
-
- override def setBoolean(ordinal: Int, value: Boolean): Unit = row.setBoolean(ordinal, value)
-
- override def setByte(ordinal: Int, value: Byte): Unit = row.setByte(ordinal, value)
-
- override def setShort(ordinal: Int, value: Short): Unit = row.setShort(ordinal, value)
-
- override def setInt(ordinal: Int, value: Int): Unit = row.setInt(ordinal, value)
-
- override def setLong(ordinal: Int, value: Long): Unit = row.setLong(ordinal, value)
-
- override def setDouble(ordinal: Int, value: Double): Unit = row.setDouble(ordinal, value)
-
- override def setFloat(ordinal: Int, value: Float): Unit = row.setFloat(ordinal, value)
-
- override def setDecimal(ordinal: Int, value: Decimal): Unit =
- row.setDecimal(ordinal, value, value.precision)
- }
-
- final class ArrayDataUpdater(array: ArrayData) extends CatalystDataUpdater {
- override def set(ordinal: Int, value: Any): Unit = array.update(ordinal, value)
-
- override def setNullAt(ordinal: Int): Unit = array.setNullAt(ordinal)
-
- override def setBoolean(ordinal: Int, value: Boolean): Unit = array.setBoolean(ordinal, value)
-
- override def setByte(ordinal: Int, value: Byte): Unit = array.setByte(ordinal, value)
-
- override def setShort(ordinal: Int, value: Short): Unit = array.setShort(ordinal, value)
-
- override def setInt(ordinal: Int, value: Int): Unit = array.setInt(ordinal, value)
-
- override def setLong(ordinal: Int, value: Long): Unit = array.setLong(ordinal, value)
-
- override def setDouble(ordinal: Int, value: Double): Unit = array.setDouble(ordinal, value)
-
- override def setFloat(ordinal: Int, value: Float): Unit = array.setFloat(ordinal, value)
-
- override def setDecimal(ordinal: Int, value: Decimal): Unit = array.update(ordinal, value)
- }
-}
-
-object AvroDeserializer {
-
- // NOTE: Following methods have been renamed in Spark 3.1.3 [1] making [[AvroDeserializer]] implementation
- // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]].
- // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.1.x branch,
- // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.1.2 as well as
- // w/ Spark >= 3.1.3
- //
- // [1] https://github.com/apache/spark/pull/34978
-
- def createDateRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Int => Int = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
- if (days < RebaseDateTime.lastSwitchJulianDay) {
- throw DataSourceUtils.newRebaseExceptionInRead(format)
- }
- days
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays
- case LegacyBehaviorPolicy.CORRECTED => identity[Int]
- }
-
- def createTimestampRebaseFuncInRead(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Long => Long = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
- if (micros < RebaseDateTime.lastSwitchJulianTs) {
- throw DataSourceUtils.newRebaseExceptionInRead(format)
- }
- micros
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianMicros
- case LegacyBehaviorPolicy.CORRECTED => identity[Long]
- }
-}
-
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
deleted file mode 100644
index e35b9271012..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ /dev/null
@@ -1,389 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Conversions.DecimalConversion
-import org.apache.avro.LogicalTypes.{TimestampMicros, TimestampMillis}
-import org.apache.avro.{LogicalTypes, Schema}
-import org.apache.avro.Schema.Type
-import org.apache.avro.Schema.Type._
-import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record}
-import org.apache.avro.util.Utf8
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.avro.AvroSerializer.{createDateRebaseFuncInWrite, createTimestampRebaseFuncInWrite}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, RebaseDateTime}
-import org.apache.spark.sql.execution.datasources.DataSourceUtils
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types._
-
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-
-/**
- * A serializer to serialize data in catalyst format to data in avro format.
- *
- * NOTE: This code is borrowed from Spark 3.1.2
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[sql] class AvroSerializer(rootCatalystType: DataType,
- rootAvroType: Schema,
- nullable: Boolean,
- datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging {
-
- def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) = {
- this(rootCatalystType, rootAvroType, nullable,
- LegacyBehaviorPolicy.withName(SQLConf.get.getConf(
- SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE, LegacyBehaviorPolicy.CORRECTED.toString)))
- }
-
- def serialize(catalystData: Any): Any = {
- converter.apply(catalystData)
- }
-
- private val dateRebaseFunc = createDateRebaseFuncInWrite(
- datetimeRebaseMode, "Avro")
-
- private val timestampRebaseFunc = createTimestampRebaseFuncInWrite(
- datetimeRebaseMode, "Avro")
-
- private val converter: Any => Any = {
- val actualAvroType = resolveNullableType(rootAvroType, nullable)
- val baseConverter = rootCatalystType match {
- case st: StructType =>
- newStructConverter(st, actualAvroType).asInstanceOf[Any => Any]
- case _ =>
- val tmpRow = new SpecificInternalRow(Seq(rootCatalystType))
- val converter = newConverter(rootCatalystType, actualAvroType)
- (data: Any) =>
- tmpRow.update(0, data)
- converter.apply(tmpRow, 0)
- }
- if (nullable) {
- (data: Any) =>
- if (data == null) {
- null
- } else {
- baseConverter.apply(data)
- }
- } else {
- baseConverter
- }
- }
-
- private type Converter = (SpecializedGetters, Int) => Any
-
- private lazy val decimalConversions = new DecimalConversion()
-
- private def newConverter(catalystType: DataType, avroType: Schema): Converter = {
- (catalystType, avroType.getType) match {
- case (NullType, NULL) =>
- (getter, ordinal) => null
- case (BooleanType, BOOLEAN) =>
- (getter, ordinal) => getter.getBoolean(ordinal)
- case (ByteType, INT) =>
- (getter, ordinal) => getter.getByte(ordinal).toInt
- case (ShortType, INT) =>
- (getter, ordinal) => getter.getShort(ordinal).toInt
- case (IntegerType, INT) =>
- (getter, ordinal) => getter.getInt(ordinal)
- case (LongType, LONG) =>
- (getter, ordinal) => getter.getLong(ordinal)
- case (FloatType, FLOAT) =>
- (getter, ordinal) => getter.getFloat(ordinal)
- case (DoubleType, DOUBLE) =>
- (getter, ordinal) => getter.getDouble(ordinal)
- case (d: DecimalType, FIXED)
- if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) =>
- (getter, ordinal) =>
- val decimal = getter.getDecimal(ordinal, d.precision, d.scale)
- decimalConversions.toFixed(decimal.toJavaBigDecimal, avroType,
- LogicalTypes.decimal(d.precision, d.scale))
-
- case (d: DecimalType, BYTES)
- if avroType.getLogicalType == LogicalTypes.decimal(d.precision, d.scale) =>
- (getter, ordinal) =>
- val decimal = getter.getDecimal(ordinal, d.precision, d.scale)
- decimalConversions.toBytes(decimal.toJavaBigDecimal, avroType,
- LogicalTypes.decimal(d.precision, d.scale))
-
- case (StringType, ENUM) =>
- val enumSymbols: Set[String] = avroType.getEnumSymbols.asScala.toSet
- (getter, ordinal) =>
- val data = getter.getUTF8String(ordinal).toString
- if (!enumSymbols.contains(data)) {
- throw new IncompatibleSchemaException(
- "Cannot write \"" + data + "\" since it's not defined in enum \"" +
- enumSymbols.mkString("\", \"") + "\"")
- }
- new EnumSymbol(avroType, data)
-
- case (StringType, STRING) =>
- (getter, ordinal) => new Utf8(getter.getUTF8String(ordinal).getBytes)
-
- case (BinaryType, FIXED) =>
- val size = avroType.getFixedSize()
- (getter, ordinal) =>
- val data: Array[Byte] = getter.getBinary(ordinal)
- if (data.length != size) {
- throw new IncompatibleSchemaException(
- s"Cannot write ${data.length} ${if (data.length > 1) "bytes" else "byte"} of " +
- "binary data into FIXED Type with size of " +
- s"$size ${if (size > 1) "bytes" else "byte"}")
- }
- new Fixed(avroType, data)
-
- case (BinaryType, BYTES) =>
- (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
-
- case (DateType, INT) =>
- (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal))
-
- case (TimestampType, LONG) => avroType.getLogicalType match {
- // For backward compatibility, if the Avro type is Long and it is not logical type
- // (the `null` case), output the timestamp value as with millisecond precision.
- case null | _: TimestampMillis => (getter, ordinal) =>
- DateTimeUtils.microsToMillis(timestampRebaseFunc(getter.getLong(ordinal)))
- case _: TimestampMicros => (getter, ordinal) =>
- timestampRebaseFunc(getter.getLong(ordinal))
- case other => throw new IncompatibleSchemaException(
- s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
- }
-
- case (ArrayType(et, containsNull), ARRAY) =>
- val elementConverter = newConverter(
- et, resolveNullableType(avroType.getElementType, containsNull))
- (getter, ordinal) => {
- val arrayData = getter.getArray(ordinal)
- val len = arrayData.numElements()
- val result = new Array[Any](len)
- var i = 0
- while (i < len) {
- if (containsNull && arrayData.isNullAt(i)) {
- result(i) = null
- } else {
- result(i) = elementConverter(arrayData, i)
- }
- i += 1
- }
- // avro writer is expecting a Java Collection, so we convert it into
- // `ArrayList` backed by the specified array without data copying.
- java.util.Arrays.asList(result: _*)
- }
-
- case (st: StructType, RECORD) =>
- val structConverter = newStructConverter(st, avroType)
- val numFields = st.length
- (getter, ordinal) => structConverter(getter.getStruct(ordinal, numFields))
-
- case (st: StructType, UNION) =>
- val unionConverter = newUnionConverter(st, avroType)
- val numFields = st.length
- (getter, ordinal) => unionConverter(getter.getStruct(ordinal, numFields))
-
- case (MapType(kt, vt, valueContainsNull), MAP) if kt == StringType =>
- val valueConverter = newConverter(
- vt, resolveNullableType(avroType.getValueType, valueContainsNull))
- (getter, ordinal) =>
- val mapData = getter.getMap(ordinal)
- val len = mapData.numElements()
- val result = new java.util.HashMap[String, Any](len)
- val keyArray = mapData.keyArray()
- val valueArray = mapData.valueArray()
- var i = 0
- while (i < len) {
- val key = keyArray.getUTF8String(i).toString
- if (valueContainsNull && valueArray.isNullAt(i)) {
- result.put(key, null)
- } else {
- result.put(key, valueConverter(valueArray, i))
- }
- i += 1
- }
- result
-
- case other =>
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystType to " +
- s"Avro type $avroType.")
- }
- }
-
- private def newStructConverter(catalystStruct: StructType, avroStruct: Schema): InternalRow => Record = {
- if (avroStruct.getType != RECORD || avroStruct.getFields.size() != catalystStruct.length) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " +
- s"Avro type $avroStruct.")
- }
- val avroSchemaHelper = new AvroUtils.AvroSchemaHelper(avroStruct)
-
- val (avroIndices: Array[Int], fieldConverters: Array[Converter]) =
- catalystStruct.map { catalystField =>
- val avroField = avroSchemaHelper.getFieldByName(catalystField.name) match {
- case Some(f) => f
- case None => throw new IncompatibleSchemaException(
- s"Cannot find ${catalystField.name} in Avro schema")
- }
- val converter = newConverter(catalystField.dataType, resolveNullableType(
- avroField.schema(), catalystField.nullable))
- (avroField.pos(), converter)
- }.toArray.unzip
-
- val numFields = catalystStruct.length
- row: InternalRow =>
- val result = new Record(avroStruct)
- var i = 0
- while (i < numFields) {
- if (row.isNullAt(i)) {
- result.put(avroIndices(i), null)
- } else {
- result.put(avroIndices(i), fieldConverters(i).apply(row, i))
- }
- i += 1
- }
- result
- }
-
- private def newUnionConverter(catalystStruct: StructType, avroUnion: Schema): InternalRow => Any = {
- if (avroUnion.getType != UNION || !canMapUnion(catalystStruct, avroUnion)) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst type $catalystStruct to " +
- s"Avro type $avroUnion.")
- }
- val nullable = avroUnion.getTypes.size() > 0 && avroUnion.getTypes.get(0).getType == Type.NULL
- val avroInnerTypes = if (nullable) {
- avroUnion.getTypes.asScala.tail
- } else {
- avroUnion.getTypes.asScala
- }
- val fieldConverters = catalystStruct.zip(avroInnerTypes).map {
- case (f1, f2) => newConverter(f1.dataType, f2)
- }
- val numFields = catalystStruct.length
- (row: InternalRow) =>
- var i = 0
- var result: Any = null
- while (i < numFields) {
- if (!row.isNullAt(i)) {
- if (result != null) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " +
- s"Avro union $avroUnion. Record has more than one optional values set")
- }
- result = fieldConverters(i).apply(row, i)
- }
- i += 1
- }
- if (!nullable && result == null) {
- throw new IncompatibleSchemaException(s"Cannot convert Catalyst record $catalystStruct to " +
- s"Avro union $avroUnion. Record has no values set, while should have exactly one")
- }
- result
- }
-
- private def canMapUnion(catalystStruct: StructType, avroStruct: Schema): Boolean = {
- (avroStruct.getTypes.size() > 0 &&
- avroStruct.getTypes.get(0).getType == Type.NULL &&
- avroStruct.getTypes.size() - 1 == catalystStruct.length) || avroStruct.getTypes.size() == catalystStruct.length
- }
-
- /**
- * Resolve a possibly nullable Avro Type.
- *
- * An Avro type is nullable when it is a [[UNION]] of two types: one null type and another
- * non-null type. This method will check the nullability of the input Avro type and return the
- * non-null type within when it is nullable. Otherwise it will return the input Avro type
- * unchanged. It will throw an [[UnsupportedAvroTypeException]] when the input Avro type is an
- * unsupported nullable type.
- *
- * It will also log a warning message if the nullability for Avro and catalyst types are
- * different.
- */
- private def resolveNullableType(avroType: Schema, nullable: Boolean): Schema = {
- val (avroNullable, resolvedAvroType) = resolveAvroType(avroType)
- warnNullabilityDifference(avroNullable, nullable)
- resolvedAvroType
- }
-
- /**
- * Check the nullability of the input Avro type and resolve it when it is nullable. The first
- * return value is a [[Boolean]] indicating if the input Avro type is nullable. The second
- * return value is the possibly resolved type.
- */
- private def resolveAvroType(avroType: Schema): (Boolean, Schema) = {
- if (avroType.getType == Type.UNION) {
- val fields = avroType.getTypes.asScala
- val actualType = fields.filter(_.getType != Type.NULL)
- if (fields.length == 2 && actualType.length == 1) {
- (true, actualType.head)
- } else {
- // This is just a normal union, not used to designate nullability
- (false, avroType)
- }
- } else {
- (false, avroType)
- }
- }
-
- /**
- * log a warning message if the nullability for Avro and catalyst types are different.
- */
- private def warnNullabilityDifference(avroNullable: Boolean, catalystNullable: Boolean): Unit = {
- if (avroNullable && !catalystNullable) {
- logWarning("Writing Avro files with nullable Avro schema and non-nullable catalyst schema.")
- }
- if (!avroNullable && catalystNullable) {
- logWarning("Writing Avro files with non-nullable Avro schema and nullable catalyst " +
- "schema will throw runtime exception if there is a record with null value.")
- }
- }
-}
-
-object AvroSerializer {
-
- // NOTE: Following methods have been renamed in Spark 3.1.3 [1] making [[AvroDeserializer]] implementation
- // (which relies on it) be only compatible with the exact same version of [[DataSourceUtils]].
- // To make sure this implementation is compatible w/ all Spark versions w/in Spark 3.1.x branch,
- // we're preemptively cloned those methods to make sure Hudi is compatible w/ Spark 3.1.2 as well as
- // w/ Spark >= 3.1.3
- //
- // [1] https://github.com/apache/spark/pull/34978
-
- def createDateRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Int => Int = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
- if (days < RebaseDateTime.lastSwitchGregorianDay) {
- throw DataSourceUtils.newRebaseExceptionInWrite(format)
- }
- days
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays
- case LegacyBehaviorPolicy.CORRECTED => identity[Int]
- }
-
- def createTimestampRebaseFuncInWrite(rebaseMode: LegacyBehaviorPolicy.Value,
- format: String): Long => Long = rebaseMode match {
- case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
- if (micros < RebaseDateTime.lastSwitchGregorianTs) {
- throw DataSourceUtils.newRebaseExceptionInWrite(format)
- }
- micros
- case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
- case LegacyBehaviorPolicy.CORRECTED => identity[Long]
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
deleted file mode 100644
index 54eacbaa0a1..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.internal.SQLConf
-
-import java.util.Locale
-import scala.collection.JavaConverters._
-
-/**
- * NOTE: This code is borrowed from Spark 3.1.3
- * This code is borrowed, so that we can better control compatibility w/in Spark minor
- * branches (3.2.x, 3.1.x, etc)
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-private[avro] object AvroUtils extends Logging {
-
- /**
- * Wraps an Avro Schema object so that field lookups are faster.
- *
- * @param avroSchema The schema in which to search for fields. Must be of type RECORD.
- */
- class AvroSchemaHelper(avroSchema: Schema) {
- if (avroSchema.getType != Schema.Type.RECORD) {
- throw new IncompatibleSchemaException(
- s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}")
- }
-
- private[this] val fieldMap = avroSchema.getFields.asScala
- .groupBy(_.name.toLowerCase(Locale.ROOT))
- .mapValues(_.toSeq) // toSeq needed for scala 2.13
-
- /**
- * Extract a single field from the contained avro schema which has the desired field name,
- * performing the matching with proper case sensitivity according to SQLConf.resolver.
- *
- * @param name The name of the field to search for.
- * @return `Some(match)` if a matching Avro field is found, otherwise `None`.
- */
- def getFieldByName(name: String): Option[Schema.Field] = {
-
- // get candidates, ignoring case of field name
- val candidates = fieldMap.get(name.toLowerCase(Locale.ROOT))
- .getOrElse(Seq.empty[Schema.Field])
-
- // search candidates, taking into account case sensitivity settings
- candidates.filter(f => SQLConf.get.resolver(f.name(), name)) match {
- case Seq(avroField) => Some(avroField)
- case Seq() => None
- case matches => throw new IncompatibleSchemaException(
- s"Searching for '$name' in Avro schema gave ${matches.size} matches. Candidates: " +
- matches.map(_.name()).mkString("[", ", ", "]")
- )
- }
- }
- }
-
- /**
- * Extract a single field from `avroSchema` which has the desired field name,
- * performing the matching with proper case sensitivity according to [[SQLConf.resolver]].
- *
- * @param avroSchema The schema in which to search for the field. Must be of type RECORD.
- * @param name The name of the field to search for.
- * @return `Some(match)` if a matching Avro field is found, otherwise `None`.
- * @throws IncompatibleSchemaException if `avroSchema` is not a RECORD or contains multiple
- * fields matching `name` (i.e., case-insensitive matching
- * is used and `avroSchema` has two or more fields that have
- * the same name with difference case).
- */
- private[avro] def getAvroFieldByName(
- avroSchema: Schema,
- name: String): Option[Schema.Field] = {
- if (avroSchema.getType != Schema.Type.RECORD) {
- throw new IncompatibleSchemaException(
- s"Attempting to treat ${avroSchema.getName} as a RECORD, but it was: ${avroSchema.getType}")
- }
- avroSchema.getFields.asScala.filter(f => SQLConf.get.resolver(f.name(), name)).toSeq match {
- case Seq(avroField) => Some(avroField)
- case Seq() => None
- case matches => throw new IncompatibleSchemaException(
- s"Searching for '$name' in Avro schema gave ${matches.size} matches. Candidates: " +
- matches.map(_.name()).mkString("[", ", ", "]")
- )
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala
deleted file mode 100644
index 63f5f9407e7..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.sql.catalyst.NoopFilters
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types.DataType
-
-class HoodieSpark3_1AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType)
- extends HoodieAvroDeserializer {
-
- private val avroDeserializer = {
- val avroRebaseModeInRead = LegacyBehaviorPolicy
- .withName(SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ, LegacyBehaviorPolicy.CORRECTED.toString))
- new AvroDeserializer(rootAvroType, rootCatalystType, avroRebaseModeInRead, new NoopFilters)
- }
-
-
- def deserialize(data: Any): Option[Any] = avroDeserializer.deserialize(data)
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroSerializer.scala
deleted file mode 100644
index 942a4e1b3b5..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroSerializer.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.avro
-
-import org.apache.avro.Schema
-import org.apache.spark.sql.types.DataType
-
-class HoodieSpark3_1AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean)
- extends HoodieAvroSerializer {
-
- val avroSerializer = new AvroSerializer(rootCatalystType, rootAvroType, nullable)
-
- override def serialize(catalystData: Any): Any = avroSerializer.serialize(catalystData)
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark31Analysis.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark31Analysis.scala
deleted file mode 100644
index 5f03faf63a8..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark31Analysis.scala
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolveLambdaVariables, UnresolvedAttribute, UnresolvedExtractValue, caseInsensitiveResolution, withPosition}
-import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentDate, CurrentTimestamp, Expression, ExtractValue, GetStructField, LambdaFunction}
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.util.toPrettySQL
-import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
-
-/**
- * NOTE: Taken from HoodieSpark2Analysis applied to Spark version 3.1.3 and modified to resolve source and target tables
- * if not already resolved
- *
- * PLEASE REFRAIN MAKING ANY CHANGES TO THIS CODE UNLESS ABSOLUTELY NECESSARY
- */
-object HoodieSpark31Analysis {
-
- case class ResolveReferences(spark: SparkSession) extends Rule[LogicalPlan] {
-
- private val resolver = spark.sessionState.conf.resolver
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
- case mO @ MergeIntoTable(targetTableO, sourceTableO, _, _, _)
- // START: custom Hudi change: don't want to go to the spark mit resolution so we resolve the source and target if they haven't been
- if !mO.resolved || containsUnresolvedStarAssignments(mO) =>
- lazy val analyzer = spark.sessionState.analyzer
- val targetTable = if (targetTableO.resolved) targetTableO else analyzer.execute(targetTableO)
- val sourceTable = if (sourceTableO.resolved) sourceTableO else analyzer.execute(sourceTableO)
- val m = mO.copy(targetTable = targetTable, sourceTable = sourceTable)
- // END: custom Hudi change.
- EliminateSubqueryAliases(targetTable) match {
- case _ =>
- val newMatchedActions = m.matchedActions.map {
- case DeleteAction(deleteCondition) =>
- val resolvedDeleteCondition = deleteCondition.map(resolveExpressionTopDown(_, m))
- DeleteAction(resolvedDeleteCondition)
- case UpdateAction(updateCondition, assignments) =>
- val resolvedUpdateCondition = updateCondition.map(resolveExpressionTopDown(_, m))
- // The update value can access columns from both target and source tables.
- UpdateAction(
- resolvedUpdateCondition,
- resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false))
- case o => o
- }
- val newNotMatchedActions = m.notMatchedActions.map {
- case InsertAction(insertCondition, assignments) =>
- // The insert action is used when not matched, so its condition and value can only
- // access columns from the source table.
- val resolvedInsertCondition =
- insertCondition.map(resolveExpressionTopDown(_, Project(Nil, m.sourceTable)))
- InsertAction(
- resolvedInsertCondition,
- resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true))
- case o => o
- }
- val resolvedMergeCondition = resolveExpressionTopDown(m.mergeCondition, m)
- m.copy(mergeCondition = resolvedMergeCondition,
- matchedActions = newMatchedActions,
- notMatchedActions = newNotMatchedActions)
- }
- }
-
- private def resolveAssignments(assignments: Seq[Assignment],
- mergeInto: MergeIntoTable,
- resolveValuesWithSourceOnly: Boolean): Seq[Assignment] = {
- if (assignments.isEmpty) {
- ////Hudi change: filter out meta fields
- //
- val expandedColumns = HoodieSqlCommonUtils.removeMetaFields(mergeInto.targetTable.output)
- //
- ////
- val expandedValues = mergeInto.sourceTable.output
- expandedColumns.zip(expandedValues).map(kv => Assignment(kv._1, kv._2))
- } else {
- assignments.map { assign =>
- val resolvedKey = assign.key match {
- case c if !c.resolved =>
- resolveExpressionTopDown(c, Project(Nil, mergeInto.targetTable))
- case o => o
- }
- val resolvedValue = assign.value match {
- // The update values may contain target and/or source references.
- case c if !c.resolved =>
- if (resolveValuesWithSourceOnly) {
- resolveExpressionTopDown(c, Project(Nil, mergeInto.sourceTable))
- } else {
- resolveExpressionTopDown(c, mergeInto)
- }
- case o => o
- }
- Assignment(resolvedKey, resolvedValue)
- }
- }
- }
-
- /**
- * Resolves the attribute and extract value expressions(s) by traversing the
- * input expression in top down manner. The traversal is done in top-down manner as
- * we need to skip over unbound lambda function expression. The lambda expressions are
- * resolved in a different rule [[ResolveLambdaVariables]]
- *
- * Example :
- * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
- *
- * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]]
- *
- * Note : In this routine, the unresolved attributes are resolved from the input plan's
- * children attributes.
- *
- * @param e The expression need to be resolved.
- * @param q The LogicalPlan whose children are used to resolve expression's attribute.
- * @param trimAlias When true, trim unnecessary alias of `GetStructField`. Note that,
- * we cannot trim the alias of top-level `GetStructField`, as we should
- * resolve `UnresolvedAttribute` to a named expression. The caller side
- * can trim the alias of top-level `GetStructField` if it's safe to do so.
- * @return resolved Expression.
- */
- private def resolveExpressionTopDown(e: Expression,
- q: LogicalPlan,
- trimAlias: Boolean = false): Expression = {
-
- def innerResolve(e: Expression, isTopLevel: Boolean): Expression = {
- // scalastyle:off return
- if (e.resolved) return e
- // scalastyle:on return
- e match {
- case f: LambdaFunction if !f.bound => f
- case u@UnresolvedAttribute(nameParts) =>
- // Leave unchanged if resolution fails. Hopefully will be resolved next round.
- val resolved =
- withPosition(u) {
- q.resolveChildren(nameParts, resolver)
- .orElse(resolveLiteralFunction(nameParts, u, q))
- .getOrElse(u)
- }
- val result = resolved match {
- // As the comment of method `resolveExpressionTopDown`'s param `trimAlias` said,
- // when trimAlias = true, we will trim unnecessary alias of `GetStructField` and
- // we won't trim the alias of top-level `GetStructField`. Since we will call
- // CleanupAliases later in Analyzer, trim non top-level unnecessary alias of
- // `GetStructField` here is safe.
- case Alias(s: GetStructField, _) if trimAlias && !isTopLevel => s
- case others => others
- }
- logDebug(s"Resolving $u to $result")
- result
- case UnresolvedExtractValue(child, fieldExpr) if child.resolved =>
- ExtractValue(child, fieldExpr, resolver)
- case _ => e.mapChildren(innerResolve(_, isTopLevel = false))
- }
- }
-
- innerResolve(e, isTopLevel = true)
- }
-
- /**
- * Literal functions do not require the user to specify braces when calling them
- * When an attributes is not resolvable, we try to resolve it as a literal function.
- */
- private def resolveLiteralFunction(nameParts: Seq[String],
- attribute: UnresolvedAttribute,
- plan: LogicalPlan): Option[Expression] = {
- // scalastyle:off return
- if (nameParts.length != 1) return None
- // scalastyle:on return
- val isNamedExpression = plan match {
- case Aggregate(_, aggregateExpressions, _) => aggregateExpressions.contains(attribute)
- case Project(projectList, _) => projectList.contains(attribute)
- case Window(windowExpressions, _, _, _) => windowExpressions.contains(attribute)
- case _ => false
- }
- val wrapper: Expression => Expression =
- if (isNamedExpression) f => Alias(f, toPrettySQL(f))() else identity
- // support CURRENT_DATE and CURRENT_TIMESTAMP
- val literalFunctions = Seq(CurrentDate(), CurrentTimestamp())
- val name = nameParts.head
- val func = literalFunctions.find(e => caseInsensitiveResolution(e.prettyName, name))
- func.map(wrapper)
- }
-
- // START: custom Hudi change: Following section is amended to the original (Spark's) implementation
- private def containsUnresolvedStarAssignments(mit: MergeIntoTable): Boolean = {
- val containsUnresolvedInsertStar = mit.notMatchedActions.exists {
- case InsertAction(_, assignments) => assignments.isEmpty
- case _ => false
- }
- val containsUnresolvedUpdateStar = mit.matchedActions.exists {
- case UpdateAction(_, assignments) => assignments.isEmpty
- case _ => false
- }
-
- containsUnresolvedInsertStar || containsUnresolvedUpdateStar
- }
- // END: custom Hudi change.
- }
-
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
deleted file mode 100644
index f78891fa61f..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
-import org.apache.spark.sql.types.DataType
-
-/**
- * ALTER TABLE ... ADD COLUMNS command, as parsed from SQL.
- */
-case class HoodieAlterTableAddColumnsStatement(
- tableName: Seq[String],
- columnsToAdd: Seq[QualifiedColType]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... CHANGE COLUMN command, as parsed from SQL.
- */
-case class HoodieAlterTableAlterColumnStatement(
- tableName: Seq[String],
- column: Seq[String],
- dataType: Option[DataType],
- nullable: Option[Boolean],
- comment: Option[String],
- position: Option[ColumnPosition]) extends ParsedStatement
-
-
-/**
- * ALTER TABLE ... RENAME COLUMN command, as parsed from SQL.
- */
-case class HoodieAlterTableRenameColumnStatement(
- tableName: Seq[String],
- column: Seq[String],
- newName: String) extends ParsedStatement
-
-/**
- * ALTER TABLE ... DROP COLUMNS command, as parsed from SQL.
- */
-case class HoodieAlterTableDropColumnsStatement(
- tableName: Seq[String], columnsToDrop: Seq[Seq[String]]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... SET TBLPROPERTIES command, as parsed from SQL.
- */
-case class HoodieAlterTableSetPropertiesStatement(
- tableName: Seq[String], properties: Map[String, String]) extends ParsedStatement
-
-/**
- * ALTER TABLE ... UNSET TBLPROPERTIES command, as parsed from SQL.
- */
-case class HoodieAlterTableUnsetPropertiesStatement(
- tableName: Seq[String], propertyKeys: Seq[String], ifExists: Boolean) extends ParsedStatement
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala
deleted file mode 100644
index 64a6d8b8fa0..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources
-
-import org.apache.hudi.storage.StoragePath
-
-import org.apache.hadoop.fs.FileStatus
-import org.apache.spark.sql.catalyst.InternalRow
-
-/**
- * Utils on Spark [[PartitionedFile]] and [[PartitionDirectory]] for Spark 3.1.
- */
-object HoodieSpark31PartitionedFileUtils extends HoodieSparkPartitionedFileUtils {
- override def getPathFromPartitionedFile(partitionedFile: PartitionedFile): StoragePath = {
- new StoragePath(partitionedFile.filePath)
- }
-
- override def getStringPathFromPartitionedFile(partitionedFile: PartitionedFile): String = {
- partitionedFile.filePath
- }
-
- override def createPartitionedFile(partitionValues: InternalRow,
- filePath: StoragePath,
- start: Long,
- length: Long): PartitionedFile = {
- PartitionedFile(partitionValues, filePath.toUri.toString, start, length)
- }
-
- override def toFileStatuses(partitionDirs: Seq[PartitionDirectory]): Seq[FileStatus] = {
- partitionDirs.flatMap(_.files)
- }
-
- override def newPartitionDirectory(internalRow: InternalRow, statuses: Seq[FileStatus]): PartitionDirectory = {
- PartitionDirectory(internalRow, statuses)
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala
deleted file mode 100644
index 1b29c428bb8..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources
-
-import org.apache.hudi.{HoodieBaseRelation, SparkAdapterSupport}
-import org.apache.spark.sql.HoodieSpark3CatalystPlanUtils
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, AttributeSet, Expression, NamedExpression, ProjectionOverSchema}
-import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.sources.BaseRelation
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
-import org.apache.spark.sql.util.SchemaUtils.restoreOriginalOutputNames
-
-/**
- * Prunes unnecessary physical columns given a [[PhysicalOperation]] over a data source relation.
- * By "physical column", we mean a column as defined in the data source format like Parquet format
- * or ORC format. For example, in Spark SQL, a root-level Parquet column corresponds to a SQL
- * column, and a nested Parquet column corresponds to a [[StructField]].
- *
- * NOTE: This class is borrowed from Spark 3.2.1, with modifications adapting it to handle [[HoodieBaseRelation]],
- * instead of [[HadoopFsRelation]]
- */
-class Spark31NestedSchemaPruning extends Rule[LogicalPlan] {
- import org.apache.spark.sql.catalyst.expressions.SchemaPruning._
-
- override def apply(plan: LogicalPlan): LogicalPlan =
- if (conf.nestedSchemaPruningEnabled) {
- apply0(plan)
- } else {
- plan
- }
-
- private def apply0(plan: LogicalPlan): LogicalPlan =
- plan transformDown {
- case op @ PhysicalOperation(projects, filters,
- // NOTE: This is modified to accommodate for Hudi's custom relations, given that original
- // [[NestedSchemaPruning]] rule is tightly coupled w/ [[HadoopFsRelation]]
- // TODO generalize to any file-based relation
- l @ LogicalRelation(relation: HoodieBaseRelation, _, _, _))
- if relation.canPruneRelationSchema =>
-
- prunePhysicalColumns(l.output, projects, filters, relation.dataSchema,
- prunedDataSchema => {
- val prunedRelation =
- relation.updatePrunedDataSchema(prunedSchema = prunedDataSchema)
- buildPrunedRelation(l, prunedRelation)
- }).getOrElse(op)
- }
-
- /**
- * This method returns optional logical plan. `None` is returned if no nested field is required or
- * all nested fields are required.
- */
- private def prunePhysicalColumns(output: Seq[AttributeReference],
- projects: Seq[NamedExpression],
- filters: Seq[Expression],
- dataSchema: StructType,
- outputRelationBuilder: StructType => LogicalRelation): Option[LogicalPlan] = {
- val (normalizedProjects, normalizedFilters) =
- normalizeAttributeRefNames(output, projects, filters)
- val requestedRootFields = identifyRootFields(normalizedProjects, normalizedFilters)
-
- // If requestedRootFields includes a nested field, continue. Otherwise,
- // return op
- if (requestedRootFields.exists { root: RootField => !root.derivedFromAtt }) {
- val prunedDataSchema = pruneDataSchema(dataSchema, requestedRootFields)
-
- // If the data schema is different from the pruned data schema, continue. Otherwise,
- // return op. We effect this comparison by counting the number of "leaf" fields in
- // each schemata, assuming the fields in prunedDataSchema are a subset of the fields
- // in dataSchema.
- if (countLeaves(dataSchema) > countLeaves(prunedDataSchema)) {
- val planUtils = SparkAdapterSupport.sparkAdapter.getCatalystPlanUtils.asInstanceOf[HoodieSpark3CatalystPlanUtils]
-
- val prunedRelation = outputRelationBuilder(prunedDataSchema)
- val projectionOverSchema = planUtils.projectOverSchema(prunedDataSchema, AttributeSet(output))
-
- Some(buildNewProjection(projects, normalizedProjects, normalizedFilters,
- prunedRelation, projectionOverSchema))
- } else {
- None
- }
- } else {
- None
- }
- }
-
- /**
- * Normalizes the names of the attribute references in the given projects and filters to reflect
- * the names in the given logical relation. This makes it possible to compare attributes and
- * fields by name. Returns a tuple with the normalized projects and filters, respectively.
- */
- private def normalizeAttributeRefNames(output: Seq[AttributeReference],
- projects: Seq[NamedExpression],
- filters: Seq[Expression]): (Seq[NamedExpression], Seq[Expression]) = {
- val normalizedAttNameMap = output.map(att => (att.exprId, att.name)).toMap
- val normalizedProjects = projects.map(_.transform {
- case att: AttributeReference if normalizedAttNameMap.contains(att.exprId) =>
- att.withName(normalizedAttNameMap(att.exprId))
- }).map { case expr: NamedExpression => expr }
- val normalizedFilters = filters.map(_.transform {
- case att: AttributeReference if normalizedAttNameMap.contains(att.exprId) =>
- att.withName(normalizedAttNameMap(att.exprId))
- })
- (normalizedProjects, normalizedFilters)
- }
-
- /**
- * Builds the new output [[Project]] Spark SQL operator that has the `leafNode`.
- */
- private def buildNewProjection(projects: Seq[NamedExpression],
- normalizedProjects: Seq[NamedExpression],
- filters: Seq[Expression],
- prunedRelation: LogicalRelation,
- projectionOverSchema: ProjectionOverSchema): Project = {
- // Construct a new target for our projection by rewriting and
- // including the original filters where available
- val projectionChild =
- if (filters.nonEmpty) {
- val projectedFilters = filters.map(_.transformDown {
- case projectionOverSchema(expr) => expr
- })
- val newFilterCondition = projectedFilters.reduce(And)
- Filter(newFilterCondition, prunedRelation)
- } else {
- prunedRelation
- }
-
- // Construct the new projections of our Project by
- // rewriting the original projections
- val newProjects = normalizedProjects.map(_.transformDown {
- case projectionOverSchema(expr) => expr
- }).map { case expr: NamedExpression => expr }
-
- if (log.isDebugEnabled) {
- logDebug(s"New projects:\n${newProjects.map(_.treeString).mkString("\n")}")
- }
-
- Project(restoreOriginalOutputNames(newProjects, projects.map(_.name)), projectionChild)
- }
-
- /**
- * Builds a pruned logical relation from the output of the output relation and the schema of the
- * pruned base relation.
- */
- private def buildPrunedRelation(outputRelation: LogicalRelation,
- prunedBaseRelation: BaseRelation): LogicalRelation = {
- val prunedOutput = getPrunedOutput(outputRelation.output, prunedBaseRelation.schema)
- outputRelation.copy(relation = prunedBaseRelation, output = prunedOutput)
- }
-
- // Prune the given output to make it consistent with `requiredSchema`.
- private def getPrunedOutput(output: Seq[AttributeReference],
- requiredSchema: StructType): Seq[AttributeReference] = {
- // We need to replace the expression ids of the pruned relation output attributes
- // with the expression ids of the original relation output attributes so that
- // references to the original relation's output are not broken
- val outputIdMap = output.map(att => (att.name, att.exprId)).toMap
- requiredSchema
- .toAttributes
- .map {
- case att if outputIdMap.contains(att.name) =>
- att.withExprId(outputIdMap(att.name))
- case att => att
- }
- }
-
- /**
- * Counts the "leaf" fields of the given dataType. Informally, this is the
- * number of fields of non-complex data type in the tree representation of
- * [[DataType]].
- */
- private def countLeaves(dataType: DataType): Int = {
- dataType match {
- case array: ArrayType => countLeaves(array.elementType)
- case map: MapType => countLeaves(map.keyType) + countLeaves(map.valueType)
- case struct: StructType =>
- struct.map(field => countLeaves(field.dataType)).sum
- case _ => 1
- }
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
deleted file mode 100644
index 729ba95b644..00000000000
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.parquet
-
-import org.apache.hudi.HoodieSparkUtils
-import org.apache.hudi.client.utils.SparkInternalSchemaConverter
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.common.util.InternalSchemaCache
-import org.apache.hudi.common.util.StringUtils.isNullOrEmpty
-import org.apache.hudi.common.util.collection.Pair
-import org.apache.hudi.internal.schema.InternalSchema
-import org.apache.hudi.internal.schema.action.InternalSchemaMerger
-import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType}
-import org.apache.parquet.filter2.compat.FilterCompat
-import org.apache.parquet.filter2.predicate.FilterApi
-import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
-import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetRecordReader}
-import org.apache.spark.TaskContext
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.avro.AvroDeserializer
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
-import org.apache.spark.sql.catalyst.expressions.{Cast, JoinedRow}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.datasources.parquet.Spark31LegacyHoodieParquetFileFormat.{createParquetFilters, pruneInternalSchema, rebuildFilterFromParquet}
-import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType}
-import org.apache.spark.util.SerializableConfiguration
-
-import java.net.URI
-
-
-/**
- * This class is an extension of [[ParquetFileFormat]] overriding Spark-specific behavior
- * that's not possible to customize in any other way
- *
- * NOTE: This is a version of [[AvroDeserializer]] impl from Spark 3.1.2 w/ w/ the following changes applied to it:
- * <ol>
- * <li>Avoiding appending partition values to the rows read from the data file</li>
- * <li>Schema on-read</li>
- * </ol>
- */
-class Spark31LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValues: Boolean) extends ParquetFileFormat {
-
- override def buildReaderWithPartitionValues(sparkSession: SparkSession,
- dataSchema: StructType,
- partitionSchema: StructType,
- requiredSchema: StructType,
- filters: Seq[Filter],
- options: Map[String, String],
- hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = {
- hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName)
- hadoopConf.set(
- ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA,
- requiredSchema.json)
- hadoopConf.set(
- ParquetWriteSupport.SPARK_ROW_SCHEMA,
- requiredSchema.json)
- hadoopConf.set(
- SQLConf.SESSION_LOCAL_TIMEZONE.key,
- sparkSession.sessionState.conf.sessionLocalTimeZone)
- hadoopConf.setBoolean(
- SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key,
- sparkSession.sessionState.conf.nestedSchemaPruningEnabled)
- hadoopConf.setBoolean(
- SQLConf.CASE_SENSITIVE.key,
- sparkSession.sessionState.conf.caseSensitiveAnalysis)
-
- ParquetWriteSupport.setSchema(requiredSchema, hadoopConf)
-
- // Sets flags for `ParquetToSparkSchemaConverter`
- hadoopConf.setBoolean(
- SQLConf.PARQUET_BINARY_AS_STRING.key,
- sparkSession.sessionState.conf.isParquetBinaryAsString)
- hadoopConf.setBoolean(
- SQLConf.PARQUET_INT96_AS_TIMESTAMP.key,
- sparkSession.sessionState.conf.isParquetINT96AsTimestamp)
-
- val internalSchemaStr = hadoopConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA)
- // For Spark DataSource v1, there's no Physical Plan projection/schema pruning w/in Spark itself,
- // therefore it's safe to do schema projection here
- if (!isNullOrEmpty(internalSchemaStr)) {
- val prunedInternalSchemaStr =
- pruneInternalSchema(internalSchemaStr, requiredSchema)
- hadoopConf.set(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA, prunedInternalSchemaStr)
- }
-
- val broadcastedHadoopConf =
- sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
-
- // TODO: if you move this into the closure it reverts to the default values.
- // If true, enable using the custom RecordReader for parquet. This only works for
- // a subset of the types (no complex types).
- val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields)
- val sqlConf = sparkSession.sessionState.conf
- val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled
- val enableVectorizedReader: Boolean =
- sqlConf.parquetVectorizedReaderEnabled &&
- resultSchema.forall(_.dataType.isInstanceOf[AtomicType])
- val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled
- val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion
- val capacity = sqlConf.parquetVectorizedReaderBatchSize
- val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown
- // Whole stage codegen (PhysicalRDD) is able to deal with batches directly
- val returningBatch = supportBatch(sparkSession, resultSchema)
- val pushDownDate = sqlConf.parquetFilterPushDownDate
- val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp
- val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal
- val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith
- val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold
- val isCaseSensitive = sqlConf.caseSensitiveAnalysis
- val timeZoneId = Option(sqlConf.sessionLocalTimeZone)
-
- (file: PartitionedFile) => {
- assert(!shouldAppendPartitionValues || file.partitionValues.numFields == partitionSchema.size)
-
- val filePath = new Path(new URI(file.filePath))
- val split =
- new org.apache.parquet.hadoop.ParquetInputSplit(
- filePath,
- file.start,
- file.start + file.length,
- file.length,
- Array.empty,
- null)
-
- val sharedConf = broadcastedHadoopConf.value.value
-
- // Fetch internal schema
- val internalSchemaStr = sharedConf.get(SparkInternalSchemaConverter.HOODIE_QUERY_SCHEMA)
- // Internal schema has to be pruned at this point
- val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr)
-
- var shouldUseInternalSchema = !isNullOrEmpty(internalSchemaStr) && querySchemaOption.isPresent
-
- val tablePath = sharedConf.get(SparkInternalSchemaConverter.HOODIE_TABLE_PATH)
- val fileSchema = if (shouldUseInternalSchema) {
- val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
- val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
- val storage = new HoodieHadoopStorage(tablePath, sharedConf)
- InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
- } else {
- null
- }
-
- lazy val footerFileMetaData =
- ParquetFileReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData
- val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
- footerFileMetaData.getKeyValueMetaData.get,
- SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
- // Try to push down filters when filter push-down is enabled.
- val pushed = if (enableParquetFilterPushDown) {
- val parquetSchema = footerFileMetaData.getSchema
- val parquetFilters = if (HoodieSparkUtils.gteqSpark3_1_3) {
- createParquetFilters(
- parquetSchema,
- pushDownDate,
- pushDownTimestamp,
- pushDownDecimal,
- pushDownStringStartWith,
- pushDownInFilterThreshold,
- isCaseSensitive,
- datetimeRebaseMode)
- } else {
- createParquetFilters(
- parquetSchema,
- pushDownDate,
- pushDownTimestamp,
- pushDownDecimal,
- pushDownStringStartWith,
- pushDownInFilterThreshold,
- isCaseSensitive)
- }
- filters.map(rebuildFilterFromParquet(_, fileSchema, querySchemaOption.orElse(null)))
- // Collects all converted Parquet filter predicates. Notice that not all predicates can be
- // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap`
- // is used here.
- .flatMap(parquetFilters.createFilter)
- .reduceOption(FilterApi.and)
- } else {
- None
- }
-
- // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps'
- // *only* if the file was created by something other than "parquet-mr", so check the actual
- // writer here for this file. We have to do this per-file, as each file in the table may
- // have different writers.
- // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads.
- def isCreatedByParquetMr: Boolean =
- footerFileMetaData.getCreatedBy().startsWith("parquet-mr")
-
- val convertTz =
- if (timestampConversion && !isCreatedByParquetMr) {
- Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
- } else {
- None
- }
-
- val int96RebaseMode = DataSourceUtils.int96RebaseMode(
- footerFileMetaData.getKeyValueMetaData.get,
- SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_READ))
-
- val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
-
- // Clone new conf
- val hadoopAttemptConf = new Configuration(broadcastedHadoopConf.value.value)
-
- val typeChangeInfos: java.util.Map[Integer, Pair[DataType, DataType]] = if (shouldUseInternalSchema) {
- val mergedInternalSchema = new InternalSchemaMerger(fileSchema, querySchemaOption.get(), true, true).mergeSchema()
- val mergedSchema = SparkInternalSchemaConverter.constructSparkSchemaFromInternalSchema(mergedInternalSchema)
-
- hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, mergedSchema.json)
-
- SparkInternalSchemaConverter.collectTypeChangedCols(querySchemaOption.get(), mergedInternalSchema)
- } else {
- val (implicitTypeChangeInfo, sparkRequestSchema) = HoodieParquetFileFormatHelper.buildImplicitSchemaChangeInfo(hadoopAttemptConf, footerFileMetaData, requiredSchema)
- if (!implicitTypeChangeInfo.isEmpty) {
- shouldUseInternalSchema = true
- hadoopAttemptConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, sparkRequestSchema.json)
- }
- implicitTypeChangeInfo
- }
-
- val hadoopAttemptContext =
- new TaskAttemptContextImpl(hadoopAttemptConf, attemptId)
-
- // Try to push down filters when filter push-down is enabled.
- // Notice: This push-down is RowGroups level, not individual records.
- if (pushed.isDefined) {
- ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
- }
- val taskContext = Option(TaskContext.get())
- if (enableVectorizedReader) {
- val vectorizedReader =
- if (shouldUseInternalSchema) {
- new Spark31HoodieVectorizedParquetRecordReader(
- convertTz.orNull,
- datetimeRebaseMode.toString,
- int96RebaseMode.toString,
- enableOffHeapColumnVector && taskContext.isDefined,
- capacity,
- typeChangeInfos)
- } else {
- new VectorizedParquetRecordReader(
- convertTz.orNull,
- datetimeRebaseMode.toString,
- int96RebaseMode.toString,
- enableOffHeapColumnVector && taskContext.isDefined,
- capacity)
- }
-
- val iter = new RecordReaderIterator(vectorizedReader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- vectorizedReader.initialize(split, hadoopAttemptContext)
-
- // NOTE: We're making appending of the partitioned values to the rows read from the
- // data file configurable
- if (shouldAppendPartitionValues) {
- logDebug(s"Appending $partitionSchema ${file.partitionValues}")
- vectorizedReader.initBatch(partitionSchema, file.partitionValues)
- } else {
- vectorizedReader.initBatch(StructType(Nil), InternalRow.empty)
- }
-
- if (returningBatch) {
- vectorizedReader.enableReturningBatches()
- }
-
- // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy.
- iter.asInstanceOf[Iterator[InternalRow]]
- } else {
- logDebug(s"Falling back to parquet-mr")
- // ParquetRecordReader returns InternalRow
- val readSupport = new ParquetReadSupport(
- convertTz,
- enableVectorizedReader = false,
- datetimeRebaseMode,
- int96RebaseMode)
- val reader = if (pushed.isDefined && enableRecordFilter) {
- val parquetFilter = FilterCompat.get(pushed.get, null)
- new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
- } else {
- new ParquetRecordReader[InternalRow](readSupport)
- }
- val iter = new RecordReaderIterator[InternalRow](reader)
- // SPARK-23457 Register a task completion listener before `initialization`.
- taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
- reader.initialize(split, hadoopAttemptContext)
-
- val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes
- val unsafeProjection = if (typeChangeInfos.isEmpty) {
- GenerateUnsafeProjection.generate(fullSchema, fullSchema)
- } else {
- // find type changed.
- val newFullSchema = new StructType(requiredSchema.fields.zipWithIndex.map { case (f, i) =>
- if (typeChangeInfos.containsKey(i)) {
- StructField(f.name, typeChangeInfos.get(i).getRight, f.nullable, f.metadata)
- } else f
- }).toAttributes ++ partitionSchema.toAttributes
- val castSchema = newFullSchema.zipWithIndex.map { case (attr, i) =>
- if (typeChangeInfos.containsKey(i)) {
- val srcType = typeChangeInfos.get(i).getRight
- val dstType = typeChangeInfos.get(i).getLeft
- val needTimeZone = Cast.needsTimeZone(srcType, dstType)
- Cast(attr, dstType, if (needTimeZone) timeZoneId else None)
- } else attr
- }
- GenerateUnsafeProjection.generate(castSchema, newFullSchema)
- }
-
- // NOTE: We're making appending of the partitioned values to the rows read from the
- // data file configurable
- if (!shouldAppendPartitionValues || partitionSchema.length == 0) {
- // There is no partition columns
- iter.map(unsafeProjection)
- } else {
- val joinedRow = new JoinedRow()
- iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues)))
- }
- }
- }
- }
-}
-
-object Spark31LegacyHoodieParquetFileFormat {
-
- def pruneInternalSchema(internalSchemaStr: String, requiredSchema: StructType): String = {
- val querySchemaOption = SerDeHelper.fromJson(internalSchemaStr)
- if (querySchemaOption.isPresent && requiredSchema.nonEmpty) {
- val prunedSchema = SparkInternalSchemaConverter.convertAndPruneStructTypeToInternalSchema(requiredSchema, querySchemaOption.get())
- SerDeHelper.toJson(prunedSchema)
- } else {
- internalSchemaStr
- }
- }
-
- private def createParquetFilters(args: Any*): ParquetFilters = {
- // ParquetFilters bears a single ctor (in Spark 3.1)
- val ctor = classOf[ParquetFilters].getConstructors.head
- ctor.newInstance(args.map(_.asInstanceOf[AnyRef]): _*)
- .asInstanceOf[ParquetFilters]
- }
... 13416 lines suppressed ...