You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2021/06/08 06:24:46 UTC
[hudi] branch master updated (cf83f10 -> f760ec5)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from cf83f10 add BootstrapFunction to support index bootstrap (#3024)
add f760ec5 [HUDI-1659] Basic Implement Of Spark Sql Support For Hoodie (#2645)
No new revisions were added by this update.
Summary of changes:
.../hudi/client/AbstractHoodieWriteClient.java | 3 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 23 +
.../hudi/execution/HoodieLazyInsertIterable.java | 13 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 13 +-
.../org/apache/hudi/io/HoodieBootstrapHandle.java | 5 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 17 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 36 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 107 +-
.../apache/hudi/io/storage/HoodieConcatHandle.java | 2 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 19 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 10 +-
.../action/bootstrap/BootstrapRecordConsumer.java | 2 +-
.../table/action/commit/AbstractMergeHelper.java | 2 +-
.../java/org/apache/hudi/io/FlinkCreateHandle.java | 8 +-
.../hudi/table/action/commit/FlinkMergeHelper.java | 4 +-
.../hudi/table/action/commit/JavaMergeHelper.java | 4 +-
.../hudi/execution/SparkLazyInsertIterable.java | 3 +-
.../hudi/table/action/commit/SparkMergeHelper.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 2 +-
.../common/model/DefaultHoodieRecordPayload.java | 31 +-
.../hudi/common/table/HoodieTableConfig.java | 11 +
.../hudi/common/table/HoodieTableMetaClient.java | 28 +-
.../hudi/common/table/TableSchemaResolver.java | 38 +-
.../apache/hudi/common/util/CollectionUtils.java | 4 +
.../hudi/common/util/PartitionPathEncodeUtils.java | 114 ++
...ption.java => HoodieDuplicateKeyException.java} | 8 +-
hudi-integ-test/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 89 ++
hudi-spark-datasource/hudi-spark/pom.xml | 21 +-
.../org/apache/hudi/sql/IExpressionEvaluator.java | 26 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 27 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 8 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 37 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 43 +-
.../org/apache/hudi/SparkAdapterSupport.scala} | 24 +-
.../apache/spark/sql/hive/HiveClientUtils.scala | 20 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 213 ++++
.../sql/hudi/HoodieSparkSessionExtension.scala | 49 +
.../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 184 ++++
.../org/apache/spark/sql/hudi/SerDeUtils.scala | 55 +
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 313 ++++++
.../command/CreateHoodieTableAsSelectCommand.scala | 112 ++
.../hudi/command/CreateHoodieTableCommand.scala | 359 +++++++
.../hudi/command/DeleteHoodieTableCommand.scala | 84 ++
.../command/InsertIntoHoodieTableCommand.scala | 270 +++++
.../hudi/command/MergeIntoHoodieTableCommand.scala | 456 ++++++++
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 84 ++
.../hudi/command/UpdateHoodieTableCommand.scala | 119 +++
.../spark/sql/hudi/command/UuidKeyGenerator.scala | 22 +-
.../hudi/command/payload/ExpressionCodeGen.scala | 185 ++++
.../hudi/command/payload/ExpressionPayload.scala | 309 ++++++
.../sql/hudi/command/payload/SqlTypedRecord.scala | 177 ++++
.../sql/hudi/streaming/HoodieStreamSource.scala | 6 +-
.../java/org/apache/hudi/client/TestBootstrap.java | 7 +-
.../src/test/resources/sql-statements.sql | 255 +++++
.../TestConvertFilterToCatalystExpression.scala | 16 -
.../org/apache/hudi/TestHoodieFileIndex.scala | 6 +-
.../functional/HoodieSparkSqlWriterSuite.scala | 8 +-
.../apache/hudi/functional/TestSqlStatement.scala | 183 ++++
.../apache/spark/sql/hudi/TestCreateTable.scala | 275 +++++
.../apache/spark/sql/hudi/TestDeleteTable.scala | 67 ++
.../apache/spark/sql/hudi/TestHoodieSqlBase.scala | 81 ++
.../apache/spark/sql/hudi/TestInsertTable.scala | 223 ++++
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 535 ++++++++++
.../apache/spark/sql/hudi/TestUpdateTable.scala | 62 ++
hudi-spark-datasource/hudi-spark2/pom.xml | 22 +-
.../hudi-spark2/src/main/antlr4/imports/SqlBase.g4 | 1100 ++++++++++++++++++++
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 107 ++
.../apache/spark/sql/adapter/Spark2Adapter.scala | 85 ++
.../catalyst/plans/logical/DeleteFromTable.scala | 23 +-
.../sql/catalyst/plans/logical/MergeInto.scala | 66 ++
.../sql/catalyst/plans/logical/UpdateTable.scala | 25 +-
.../sql/hudi/parser/HoodieSqlAstBuilder.scala | 230 ++++
.../spark/sql/hudi/parser/HoodieSqlParser.scala | 171 +++
hudi-spark-datasource/hudi-spark3/pom.xml | 4 +-
.../apache/spark/sql/adapter/Spark3Adapter.scala | 88 ++
.../org/apache/hudi/hive/HoodieHiveClient.java | 12 +-
hudi-utilities/pom.xml | 10 +-
.../functional/TestHoodieDeltaStreamer.java | 3 +
packaging/hudi-integ-test-bundle/pom.xml | 13 +-
packaging/hudi-spark-bundle/pom.xml | 14 +-
packaging/hudi-utilities-bundle/pom.xml | 14 +-
pom.xml | 9 +
style/checkstyle-suppressions.xml | 1 +
86 files changed, 7195 insertions(+), 339 deletions(-)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
copy hudi-common/src/main/java/org/apache/hudi/exception/{HoodieNotSupportedException.java => HoodieDuplicateKeyException.java} (74%)
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieKeyGeneratorException.java => hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/sql/IExpressionEvaluator.java (64%)
copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/hudi/Spark2RowSerDe.scala => hudi-spark/src/main/scala/org/apache/hudi/SparkAdapterSupport.scala} (60%)
copy hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncTool.java => hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hive/HiveClientUtils.scala (67%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSparkSessionExtension.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableAsSelectCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala
copy hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncTool.java => hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UuidKeyGenerator.scala (67%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionCodeGen.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/SqlTypedRecord.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/sql-statements.sql
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteTable.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/antlr4/imports/SqlBase.g4
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/adapter/Spark2Adapter.scala
copy hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncTool.java => hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromTable.scala (67%)
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala
copy hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncTool.java => hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateTable.scala (67%)
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/hudi/parser/HoodieSqlAstBuilder.scala
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/hudi/parser/HoodieSqlParser.scala
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/adapter/Spark3Adapter.scala