[hudi] branch master updated (195dac9 -> 5ce45c4)

[le...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

leesf pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.


    from 195dac9  [MINOR] Disable flaky tests to unlock CI (#4592)
     add 5ce45c4  [HUDI-3172] Refactor hudi existing modules to make more code reuse in V2 Implementation (#4514)

No new revisions were added by this update.

Summary of changes:
 .../org/apache/spark/sql/hudi/SparkAdapter.scala   |   4 +-
 hudi-spark-datasource/README.md                    |  38 +++
 hudi-spark-datasource/hudi-spark-common/pom.xml    |  42 +++
 .../apache/hudi/HoodieDatasetBulkInsertHelper.java |   0
 .../java/org/apache/hudi/SparkRowWriteHelper.java  |   0
 .../SparkStreamingAsyncClusteringService.java      |   0
 .../async/SparkStreamingAsyncCompactService.java   |   0
 .../org/apache/hudi/sql/IExpressionEvaluator.java  |   0
 .../main/java/org/apache/hudi/sql/InsertMode.java  |   0
 ...org.apache.spark.sql.sources.DataSourceRegister |   0
 .../main/scala/org/apache/hudi/DefaultSource.scala |   2 +-
 .../scala/org/apache/hudi/HoodieBootstrapRDD.scala |   0
 .../org/apache/hudi/HoodieBootstrapRelation.scala  |   0
 .../org/apache/hudi/HoodieEmptyRelation.scala      |   0
 .../scala/org/apache/hudi/HoodieFileIndex.scala    |   4 +-
 .../org/apache/hudi/HoodieMergeOnReadRDD.scala     |  46 ++--
 .../org/apache/hudi/HoodieSparkSqlWriter.scala     |  10 +-
 .../org/apache/hudi/HoodieStreamingSink.scala      |   0
 .../scala/org/apache/hudi/HoodieWriterUtils.scala  |   0
 .../org/apache/hudi/IncrementalRelation.scala      |   0
 .../hudi/MergeOnReadIncrementalRelation.scala      |   0
 .../apache/hudi/MergeOnReadSnapshotRelation.scala  |   4 +-
 .../src/main/scala/org/apache/hudi/package.scala   |   0
 .../spark/sql/avro/HoodieAvroDeserializer.scala    |   0
 .../spark/sql/avro/HoodieAvroSerializer.scala      |   0
 .../sql/catalyst/catalog/HoodieCatalogTable.scala  |  26 +-
 .../spark/sql/catalyst/trees/HoodieLeafLike.scala  |   0
 .../apache/spark/sql/hive/HiveClientUtils.scala    |   0
 .../apache/spark/sql/hudi/DataSkippingUtils.scala  |   0
 .../apache/spark/sql/hudi/HoodieOptionConfig.scala |   2 +-
 .../spark/sql/hudi/HoodieSqlCommonUtils.scala}     |  64 ++---
 .../org/apache/spark/sql/hudi/SerDeUtils.scala     |   0
 .../AlterHoodieTableAddColumnsCommand.scala        |   2 -
 .../AlterHoodieTableChangeColumnCommand.scala      |   2 -
 .../AlterHoodieTableDropPartitionCommand.scala     |   5 +-
 .../command/AlterHoodieTableRenameCommand.scala    |   0
 .../hudi/command/CreateHoodieTableCommand.scala    |  10 +-
 .../hudi/command/HoodieLeafRunnableCommand.scala   |   0
 .../command/ShowHoodieTablePartitionsCommand.scala |   2 +-
 .../spark/sql/hudi/command/SqlKeyGenerator.scala   |   2 +-
 .../hudi/command/TruncateHoodieTableCommand.scala  |   0
 .../hudi/command/ValidateDuplicateKeyPayload.scala |  45 ++++
 .../hudi/command/payload/ExpressionCodeGen.scala   |   0
 .../hudi/command/payload/ExpressionPayload.scala   |   0
 .../sql/hudi/command/payload/SqlTypedRecord.scala  |   0
 .../sql/hudi/streaming/HoodieSourceOffset.scala    |   0
 .../sql/hudi/streaming/HoodieStreamSource.scala    |   0
 hudi-spark-datasource/hudi-spark/pom.xml           |  12 +
 .../sql/hudi/HoodieSparkSessionExtension.scala     |   2 +-
 .../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 289 +--------------------
 .../spark/sql/hudi/analysis/HoodieAnalysis.scala   |  28 +-
 .../hudi/command/CompactionHoodiePathCommand.scala |  11 +-
 .../command/CompactionHoodieTableCommand.scala     |   5 +-
 .../command/CompactionShowHoodieTableCommand.scala |   5 +-
 .../command/CreateHoodieTableAsSelectCommand.scala |   5 +-
 .../hudi/command/DeleteHoodieTableCommand.scala    |   7 +-
 .../sql/hudi/command/DropHoodieTableCommand.scala  |   2 +-
 .../command/InsertIntoHoodieTableCommand.scala     |  33 +--
 .../hudi/command/MergeIntoHoodieTableCommand.scala |   8 +-
 .../hudi/command/UpdateHoodieTableCommand.scala    |  10 +-
 .../apache/hudi/functional/TestMORDataSource.scala |   2 -
 .../org/apache/spark/sql/hudi/TestAlterTable.scala |   2 +-
 hudi-spark-datasource/hudi-spark2-common/pom.xml   |  19 ++
 ...org.apache.spark.sql.sources.DataSourceRegister |   2 +-
 .../Spark2DefaultSource.scala}                     |  16 +-
 .../apache/spark/sql/adapter/Spark2Adapter.scala   |   8 +-
 hudi-spark-datasource/hudi-spark3-common/pom.xml   | 247 ++++++++++++++++++
 .../apache/hudi/spark3/internal/DefaultSource.java |   0
 .../HoodieBulkInsertDataInternalWriter.java        |   0
 .../HoodieBulkInsertDataInternalWriterFactory.java |   0
 .../HoodieDataSourceInternalBatchWrite.java        |   0
 .../HoodieDataSourceInternalBatchWriteBuilder.java |   0
 .../internal/HoodieDataSourceInternalTable.java    |   0
 .../spark3/internal/HoodieWriterCommitMessage.java |   0
 .../apache/hudi/spark3/internal/ReflectUtil.java   |   0
 .../scala/org/apache/hudi/Spark3RowSerDe.scala     |   0
 .../apache/spark/sql/adapter/Spark3Adapter.scala   |  15 +-
 .../datasources/Spark3ParsePartitionUtil.scala     |   2 +-
 .../{hudi-spark3 => hudi-spark3.1.x}/pom.xml       |  11 +-
 ...org.apache.spark.sql.sources.DataSourceRegister |   2 +-
 .../org/apache/hudi/Spark3xDefaultSource.scala}    |  12 +-
 hudi-spark-datasource/hudi-spark3/pom.xml          |   5 +
 ...org.apache.spark.sql.sources.DataSourceRegister |   2 +-
 .../org/apache/hudi/Spark3DefaultSource.scala}     |  12 +-
 hudi-spark-datasource/pom.xml                      |   4 +
 hudi-utilities/pom.xml                             |   6 +
 packaging/hudi-integ-test-bundle/pom.xml           |   7 +
 packaging/hudi-spark-bundle/pom.xml                |   9 +
 packaging/hudi-utilities-bundle/pom.xml            |   6 +
 pom.xml                                            |   5 +-
 90 files changed, 620 insertions(+), 501 deletions(-)
 create mode 100644 hudi-spark-datasource/README.md
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/SparkRowWriteHelper.java (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/sql/IExpressionEvaluator.java (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/sql/InsertMode.java (100%)
 copy hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/DefaultSource.scala (99%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieEmptyRelation.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieFileIndex.scala (99%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala (92%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala (99%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/IncrementalRelation.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala (99%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/package.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/avro/HoodieAvroDeserializer.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/avro/HoodieAvroSerializer.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala (91%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/trees/HoodieLeafLike.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hive/HiveClientUtils.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala (99%)
 copy hudi-spark-datasource/{hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala => hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala} (87%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala (98%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableChangeColumnCommand.scala (98%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala (98%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala (96%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/HoodieLeafRunnableCommand.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala (98%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala (98%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala (100%)
 create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionCodeGen.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/SqlTypedRecord.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala (100%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala (100%)
 create mode 100644 hudi-spark-datasource/hudi-spark2-common/pom.xml
 copy hudi-spark-datasource/{hudi-spark => hudi-spark2}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%)
 copy hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/{spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi/Spark2DefaultSource.scala} (70%)
 create mode 100644 hudi-spark-datasource/hudi-spark3-common/pom.xml
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieBulkInsertDataInternalWriter.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieBulkInsertDataInternalWriterFactory.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWrite.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWriteBuilder.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalTable.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/hudi/Spark3RowSerDe.scala (100%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/adapter/Spark3Adapter.scala (85%)
 rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala (99%)
 copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.1.x}/pom.xml (96%)
 copy hudi-spark-datasource/{hudi-spark => hudi-spark3.1.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%)
 copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark3xDefaultSource.scala} (70%)
 rename hudi-spark-datasource/{hudi-spark => hudi-spark3}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%)
 copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi-spark3/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala} (70%)