[hudi] branch master updated (924c30c7ea -> cdaec5a8da)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


    from 924c30c7ea [HUDI-4469] Flip reuse flag to true in HoodieBackedTableMetadata to improve file listing (#6214)
     add cdaec5a8da [HUDI-4186] Support Hudi with Spark 3.3.0 (#5943)

No new revisions were added by this update.

Summary of changes:
 .github/workflows/bot.yml                          |   5 +-
 .../apache/hudi/config/HoodieStorageConfig.java    |  15 ++
 .../org/apache/hudi/config/HoodieWriteConfig.java  |   4 +
 .../HoodieSparkBootstrapSchemaProvider.java        |  18 +-
 .../storage/row/HoodieRowParquetWriteSupport.java  |   1 +
 .../scala/org/apache/hudi/HoodieSparkUtils.scala   |   2 +
 .../org/apache/hudi/SparkAdapterSupport.scala      |   4 +-
 .../org/apache/spark/sql/hudi/SparkAdapter.scala   |  48 ++++-
 .../quickstart/TestHoodieFlinkQuickstart.java      |   2 +
 hudi-examples/hudi-examples-spark/pom.xml          |   6 +
 hudi-spark-datasource/README.md                    |  17 +-
 .../org/apache/hudi/BaseFileOnlyRelation.scala     |   6 +-
 .../spark/sql/hudi/analysis/HoodieAnalysis.scala   |  36 ++--
 .../hudi/command/DeleteHoodieTableCommand.scala    |   7 +-
 .../spark/sql/parser/HoodieCommonSqlParser.scala   |   8 +
 .../spark/sql/hudi/HoodieSparkSqlTestBase.scala    |   5 +-
 .../sql/hudi/TestAlterTableDropPartition.scala     |  11 +-
 .../hudi/TestNestedSchemaPruningOptimization.scala |   1 +
 .../sql/hudi/procedure/TestCallCommandParser.scala |  11 +-
 .../org/apache/hudi/Spark2HoodieFileScanRDD.scala} |  13 +-
 .../apache/spark/sql/adapter/Spark2Adapter.scala   |  35 +++-
 .../apache/hudi/spark3/internal/ReflectUtil.java   |   2 +-
 .../spark/sql/adapter/BaseSpark3Adapter.scala      |  17 --
 .../apache/hudi/Spark31HoodieFileScanRDD.scala}    |  13 +-
 .../apache/spark/sql/adapter/Spark3_1Adapter.scala |  37 +++-
 .../datasources/Spark31NestedSchemaPruning.scala}  |   2 +-
 ... => Spark312ResolveHudiAlterTableCommand.scala} |   2 +-
 ...scala => HoodieSpark3_1ExtendedSqlParser.scala} |   2 +-
 .../{hudi-spark3 => hudi-spark3.2.x}/pom.xml       |  65 +++---
 .../src/main/antlr4/imports/SqlBase.g4             |   0
 .../apache/hudi/spark/sql/parser/HoodieSqlBase.g4  |   0
 ...Spark32HoodieVectorizedParquetRecordReader.java |   0
 ...org.apache.spark.sql.sources.DataSourceRegister |   0
 .../apache/hudi/Spark32HoodieFileScanRDD.scala}    |  13 +-
 .../org/apache/hudi/Spark3DefaultSource.scala      |   0
 .../sql/HoodieSpark32CatalystExpressionUtils.scala |   0
 .../spark/sql/HoodieSpark32CatalystPlanUtils.scala |   0
 .../apache/spark/sql/adapter/Spark3_2Adapter.scala |  34 ++-
 .../apache/spark/sql/avro/AvroDeserializer.scala   |   0
 .../org/apache/spark/sql/avro/AvroSerializer.scala |   0
 .../org/apache/spark/sql/avro/AvroUtils.scala      |   0
 .../sql/avro/HoodieSpark3_2AvroDeserializer.scala  |   0
 .../sql/avro/HoodieSpark3_2AvroSerializer.scala    |   0
 .../plans/logical/TimeTravelRelation.scala         |   0
 .../sql/connector/catalog/HoodieIdentifier.scala   |   0
 .../datasources/Spark32NestedSchemaPruning.scala}  |   2 +-
 .../parquet/Spark32DataSourceUtils.scala           |   0
 .../parquet/Spark32HoodieParquetFileFormat.scala   |   0
 .../Spark32ResolveHudiAlterTableCommand.scala}     |   2 +-
 .../sql/hudi/analysis/HoodieSpark3Analysis.scala   |   0
 .../spark/sql/hudi/catalog/BasicStagedTable.scala  |   0
 .../spark/sql/hudi/catalog/HoodieCatalog.scala     |   4 +-
 .../sql/hudi/catalog/HoodieInternalV2Table.scala   |   0
 .../spark/sql/hudi/catalog/HoodieStagedTable.scala |   0
 .../spark/sql/hudi/catalog/TableCreationMode.java  |   0
 .../spark/sql/hudi/command/AlterTableCommand.scala |   0
 .../HoodieSpark3_2ExtendedSqlAstBuilder.scala      |   0
 .../parser/HoodieSpark3_2ExtendedSqlParser.scala   |   0
 .../TestHoodieBulkInsertDataInternalWriter.java    |   0
 .../TestHoodieDataSourceInternalBatchWrite.java    |   0
 .../hudi/spark3/internal/TestReflectUtil.java      |   0
 .../test/resources/log4j-surefire-quiet.properties |   0
 .../src/test/resources/log4j-surefire.properties   |   0
 .../{hudi-spark3 => hudi-spark3.3.x}/pom.xml       |  14 +-
 .../src/main/antlr4/imports/SqlBase.g4             |   0
 .../apache/hudi/spark/sql/parser/HoodieSqlBase.g4  |   0
 ...park33HoodieVectorizedParquetRecordReader.java} |   4 +-
 ...org.apache.spark.sql.sources.DataSourceRegister |   0
 .../apache/hudi/Spark33HoodieFileScanRDD.scala}    |  13 +-
 .../org/apache/hudi/Spark3DefaultSource.scala      |   0
 .../HoodieSpark33CatalystExpressionUtils.scala}    |   6 +-
 .../sql/HoodieSpark33CatalystPlanUtils.scala}      |   2 +-
 .../apache/spark/sql/adapter/Spark3_3Adapter.scala |  80 ++++++++
 .../apache/spark/sql/avro/AvroDeserializer.scala   | 171 +++++++---------
 .../org/apache/spark/sql/avro/AvroSerializer.scala | 151 +++++---------
 .../org/apache/spark/sql/avro/AvroUtils.scala      | 228 +++++++++++++++++++++
 .../sql/avro/HoodieSpark3_3AvroDeserializer.scala} |   2 +-
 .../sql/avro/HoodieSpark3_3AvroSerializer.scala}   |   2 +-
 .../plans/logical/TimeTravelRelation.scala         |   0
 .../sql/connector/catalog/HoodieIdentifier.scala   |   0
 .../datasources/Spark33NestedSchemaPruning.scala}  |  12 +-
 .../parquet/Spark33DataSourceUtils.scala}          |   2 +-
 .../parquet/Spark33HoodieParquetFileFormat.scala}  |  18 +-
 .../Spark33ResolveHudiAlterTableCommand.scala}     |   4 +-
 .../sql/hudi/analysis/HoodieSpark3Analysis.scala   |  16 +-
 .../spark/sql/hudi/catalog/BasicStagedTable.scala  |   0
 .../spark/sql/hudi/catalog/HoodieCatalog.scala     |   2 +-
 .../sql/hudi/catalog/HoodieInternalV2Table.scala   |   0
 .../spark/sql/hudi/catalog/HoodieStagedTable.scala |   0
 .../spark/sql/hudi/catalog/TableCreationMode.java  |   0
 .../spark/sql/hudi/command/AlterTableCommand.scala |   0
 .../HoodieSpark3_3ExtendedSqlAstBuilder.scala}     |  27 ++-
 .../parser/HoodieSpark3_3ExtendedSqlParser.scala}  |   8 +-
 .../TestHoodieBulkInsertDataInternalWriter.java    |   0
 .../TestHoodieDataSourceInternalBatchWrite.java    |   0
 .../hudi/spark3/internal/TestReflectUtil.java      |   0
 .../test/resources/log4j-surefire-quiet.properties |   0
 .../src/test/resources/log4j-surefire.properties   |   0
 pom.xml                                            |  58 +++++-
 99 files changed, 881 insertions(+), 389 deletions(-)
 copy hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala => hudi-spark2/src/main/scala/org/apache/hudi/Spark2HoodieFileScanRDD.scala} (71%)
 copy hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala => hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala} (71%)
 copy hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala => hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala} (99%)
 rename hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/{ResolveHudiAlterTableCommand312.scala => Spark312ResolveHudiAlterTableCommand.scala} (99%)
 rename hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/{HoodieSpark312ExtendedSqlParser.scala => HoodieSpark3_1ExtendedSqlParser.scala} (95%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/pom.xml (92%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4 (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
 copy hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala => hudi-spark3.2.x/src/main/scala/org/apache/hudi/Spark32HoodieFileScanRDD.scala} (71%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala (61%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroSerializer.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala => hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark32NestedSchemaPruning.scala} (99%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala => hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/Spark32ResolveHudiAlterTableCommand.scala} (98%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala (99%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java (100%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/test/resources/log4j-surefire-quiet.properties (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/src/test/resources/log4j-surefire.properties (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/pom.xml (96%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4 (100%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java => hudi-spark3.3.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieVectorizedParquetRecordReader.java} (98%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
 rename hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala => hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala} (67%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala} (94%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala} (95%)
 create mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala (82%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala (73%)
 create mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala} (95%)
 copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroSerializer.scala} (94%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala} (95%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala} (99%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala} (97%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala} (95%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala (94%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala (99%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala} (99%)
 rename hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala => hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala} (95%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java (100%)
 copy {hudi-common => hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire-quiet.properties (100%)
 copy {hudi-client/hudi-client-common => hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire.properties (100%)