You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2022/04/01 20:20:54 UTC
[hudi] branch master updated (9275b8f -> 444ff49)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from 9275b8f [HUDI-3468][RFC-49] Support sync with DataHub (#5022)
add 444ff49 [RFC-33] [HUDI-2429][Stacked on HUDI-2560] Support full Schema evolution for Spark (#4910)
No new revisions were added by this update.
Summary of changes:
.../apache/hudi/client/BaseHoodieWriteClient.java | 182 +++++-
.../org/apache/hudi/config/HoodieWriteConfig.java | 50 ++
.../table/action/clean/CleanActionExecutor.java | 9 +
.../table/action/commit/HoodieMergeHelper.java | 43 +-
.../hudi/table/action/compact/HoodieCompactor.java | 24 +-
.../compact/RunCompactionActionExecutor.java | 21 +-
.../client/utils/SparkInternalSchemaConverter.java | 502 +++++++++++++++
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 40 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 13 +
hudi-common/pom.xml | 6 +
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 284 ++++++++
.../hudi/common/model/WriteOperationType.java | 6 +
.../hudi/common/table/HoodieTableMetaClient.java | 14 +
.../hudi/common/table/TableSchemaResolver.java | 66 +-
.../table/log/AbstractHoodieLogRecordReader.java | 43 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 14 +-
.../common/table/log/HoodieLogFormatReader.java | 13 +-
.../table/log/HoodieMergedLogRecordScanner.java | 13 +-
.../table/log/block/HoodieAvroDataBlock.java | 41 +-
.../common/table/log/block/HoodieDataBlock.java | 22 +
.../table/timeline/HoodieActiveTimeline.java | 42 +-
.../hudi/common/table/timeline/HoodieInstant.java | 4 +
.../hudi/common/table/timeline/HoodieTimeline.java | 17 +
.../hudi/common/util/InternalSchemaCache.java | 212 ++++++
.../schema/HoodieSchemaException.java} | 21 +-
.../hudi/internal/schema/InternalSchema.java | 291 +++++++++
.../internal/schema/InternalSchemaBuilder.java | 272 ++++++++
.../java/org/apache/hudi/internal/schema/Type.java | 78 +++
.../org/apache/hudi/internal/schema/Types.java | 716 +++++++++++++++++++++
.../schema/action/InternalSchemaChangeApplier.java | 164 +++++
.../schema/action/InternalSchemaMerger.java | 197 ++++++
.../hudi/internal/schema/action/TableChange.java | 252 ++++++++
.../hudi/internal/schema/action/TableChanges.java | 398 ++++++++++++
.../internal/schema/action/TableChangesHelper.java | 79 +++
.../convert/AvroInternalSchemaConverter.java | 436 +++++++++++++
.../io/AbstractInternalSchemaStorageManager.java | 51 ++
.../io/FileBasedInternalSchemaStorageManager.java | 184 ++++++
.../schema/utils/AvroSchemaEvolutionUtils.java | 142 ++++
.../internal/schema/utils/InternalSchemaUtils.java | 270 ++++++++
.../internal/schema/utils/SchemaChangeUtils.java | 305 +++++++++
.../hudi/internal/schema/utils/SerDeHelper.java | 351 ++++++++++
.../schema/visitor/InternalSchemaVisitor.java | 86 +++
.../internal/schema/visitor/NameToIDVisitor.java | 113 ++++
.../HoodieMetadataMergedLogRecordReader.java | 3 +-
.../hudi/internal/schema/TestSerDeHelper.java | 117 ++++
.../internal/schema/action/TestMergeSchema.java | 88 +++
.../internal/schema/action/TestTableChanges.java | 229 +++++++
.../TestFileBasedInternalSchemaStorageManager.java | 110 ++++
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 422 ++++++++++++
.../schema/utils/TestInternalSchemaUtils.java | 100 +++
...org.apache.spark.sql.sources.DataSourceRegister | 3 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 3 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 11 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 26 +-
.../org/apache/hudi/HoodieDataSourceHelper.scala | 22 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 52 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 8 +
.../org/apache/hudi/IncrementalRelation.scala | 43 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 4 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 4 +-
.../parquet/SparkHoodieParquetFileFormat.scala | 48 ++
.../AlterHoodieTableAddColumnsCommand.scala | 4 +-
.../sql/hudi/HoodieSparkSessionExtension.scala | 4 +
.../org/apache/hudi/TestHoodieSparkUtils.scala | 2 +-
.../functional/TestParquetColumnProjection.scala | 2 +-
.../apache/spark/sql/hudi/TestHoodieSqlBase.scala | 16 +
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 472 ++++++++++++++
.../apache/spark/sql/adapter/Spark2Adapter.scala | 13 +
.../spark/sql/adapter/BaseSpark3Adapter.scala | 18 +
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 38 +-
...park312HoodieVectorizedParquetRecordReader.java | 188 ++++++
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 28 +
.../sql/catalyst/plans/logical/HStatments.scala | 66 ++
.../parquet/Spark312HoodieParquetFileFormat.scala | 365 +++++++++++
.../sql/hudi/ResolveHudiAlterTableCommand312.scala | 282 ++++++++
.../sql/hudi/command/AlterTableCommand312.scala | 324 ++++++++++
.../parser/HoodieSpark312ExtendedSqlParser.scala} | 26 +-
.../sql/parser/HoodieSpark312SqlAstBuilder.scala | 125 ++++
hudi-spark-datasource/hudi-spark3/pom.xml | 78 ++-
...Spark32HoodieVectorizedParquetRecordReader.java | 191 ++++++
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 27 +
.../parquet/Spark32HoodieParquetFileFormat.scala | 349 ++++++++++
.../hudi/ResolveHudiAlterTableCommandSpark32.scala | 67 ++
.../spark/sql/hudi/command/AlterTableCommand.scala | 347 ++++++++++
packaging/hudi-integ-test-bundle/pom.xml | 1 +
packaging/hudi-spark-bundle/pom.xml | 1 +
packaging/hudi-utilities-bundle/pom.xml | 1 +
pom.xml | 10 +
89 files changed, 10296 insertions(+), 136 deletions(-)
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkInternalSchemaConverter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
copy hudi-common/src/main/java/org/apache/hudi/{exception/HoodieException.java => internal/schema/HoodieSchemaException.java} (70%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/InternalSchema.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/InternalSchemaBuilder.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/action/InternalSchemaChangeApplier.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/action/InternalSchemaMerger.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChange.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChangesHelper.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/convert/AvroInternalSchemaConverter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/io/AbstractInternalSchemaStorageManager.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/InternalSchemaUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SchemaChangeUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SerDeHelper.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/InternalSchemaVisitor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToIDVisitor.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/TestSerDeHelper.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestMergeSchema.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestTableChanges.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestInternalSchemaUtils.java
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkHoodieParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark312HoodieVectorizedParquetRecordReader.java
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark312HoodieParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommand312.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand312.scala
copy hudi-spark-datasource/{hudi-spark-common/src/main/java/org/apache/hudi/sql/IExpressionEvaluator.java => hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark312ExtendedSqlParser.scala} (59%)
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark312SqlAstBuilder.scala
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala