You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by na...@apache.org on 2020/04/16 01:06:42 UTC
[incubator-hudi] branch hudi_test_suite_refactor updated (a096128
-> 1689d01)
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
from a096128 more fixes
add c0f96e0 [HUDI-687] Stop incremental reader on RO table when there is a pending compaction (#1396)
add a464a29 [HUDI-700]Add unit test for FileSystemViewCommand (#1490)
add 5d717a2 [HUDI-782] Add support of Aliyun object storage service. (#1506)
add 6d7ca2c [HUDI-727]: Copy default values of fields if not present when rewriting incoming record with new schema (#1427)
add 447ba3b [MINOR] Disabling flaky test in InlineFileSystem (#1510)
add 17bf930 [HUDI-770] Organize upsert/insert API implementation under a single package (#1495)
add 661b0b3 [HUDI-761] Refactoring rollback and restore actions using the ActionExecutor abstraction (#1492)
add 644c1cc [HUDI-698]Add unit test for CleansCommand (#1449)
add 14d4fea [HUDI-759] Integrate checkpoint provider with delta streamer (#1486)
add d65efe6 [HUDI-780] Migrate test cases to Junit 5 (#1504)
add 9ca710c [HUDI-777] Updated description for --target-table parameter (#1519)
add 1689d01 Merge branch 'master' into hudi_test_suite_refactor
No new revisions were added by this update.
Summary of changes:
hudi-cli/pom.xml | 56 +++
.../apache/hudi/cli/HoodieTableHeaderFields.java | 65 +++
.../apache/hudi/cli/commands/CleansCommand.java | 15 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 47 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 183 ++++++++
.../cli/commands/TestFileSystemViewCommand.java | 267 ++++++++++++
.../apache/hudi/cli/integ/ITTestCleansCommand.java | 101 +++++
.../src/test/resources/clean.properties | 5 +-
hudi-client/pom.xml | 24 ++
.../hudi/client/AbstractHoodieWriteClient.java | 138 +-----
.../org/apache/hudi/client/HoodieWriteClient.java | 475 +++++----------------
...kException.java => HoodieRestoreException.java} | 8 +-
.../apache/hudi/table/HoodieCopyOnWriteTable.java | 428 +++----------------
.../apache/hudi/table/HoodieMergeOnReadTable.java | 376 +++-------------
.../java/org/apache/hudi/table/HoodieTable.java | 131 ++++--
.../hudi/table/action/BaseActionExecutor.java | 5 +-
.../table/action/clean/CleanActionExecutor.java | 8 +-
.../action/commit/BaseCommitActionExecutor.java | 291 +++++++++++++
.../hudi/table/action/commit/BucketInfo.java | 28 +-
.../hudi/table/action/commit/BucketType.java | 6 +-
.../commit/BulkInsertCommitActionExecutor.java | 60 +++
.../hudi/table/action/commit/BulkInsertHelper.java | 84 ++++
.../BulkInsertPreppedCommitActionExecutor.java | 61 +++
.../table/action/commit/CommitActionExecutor.java | 176 ++++++++
.../action/commit/DeleteCommitActionExecutor.java} | 31 +-
.../hudi/table/action/commit/DeleteHelper.java | 96 +++++
.../table/action/commit/HoodieWriteMetadata.java | 104 +++++
.../hudi/table/action/commit/InsertBucket.java | 30 +-
.../action/commit/InsertCommitActionExecutor.java} | 33 +-
.../commit/InsertPreppedCommitActionExecutor.java} | 31 +-
.../apache/hudi/table/action/commit/SmallFile.java | 29 +-
.../action/commit/UpsertCommitActionExecutor.java} | 33 +-
.../table/action/commit/UpsertPartitioner.java | 316 ++++++++++++++
.../commit/UpsertPreppedCommitActionExecutor.java} | 31 +-
.../hudi/table/action/commit/WriteHelper.java | 105 +++++
.../BulkInsertDeltaCommitActionExecutor.java | 62 +++
...BulkInsertPreppedDeltaCommitActionExecutor.java | 63 +++
.../DeleteDeltaCommitActionExecutor.java} | 33 +-
.../deltacommit/DeltaCommitActionExecutor.java | 94 ++++
.../InsertDeltaCommitActionExecutor.java | 49 +++
.../InsertPreppedDeltaCommitActionExecutor.java} | 31 +-
.../UpsertDeltaCommitActionExecutor.java | 49 +++
.../deltacommit/UpsertDeltaCommitPartitioner.java | 142 ++++++
.../UpsertPreppedDeltaCommitActionExecutor.java} | 31 +-
.../action/restore/BaseRestoreActionExecutor.java | 111 +++++
.../restore/CopyOnWriteRestoreActionExecutor.java | 57 +++
.../restore/MergeOnReadRestoreActionExecutor.java | 64 +++
.../rollback/BaseRollbackActionExecutor.java | 192 +++++++++
.../CopyOnWriteRollbackActionExecutor.java | 94 ++++
.../MergeOnReadRollbackActionExecutor.java | 241 +++++++++++
.../{ => action}/rollback/RollbackHelper.java | 2 +-
.../{ => action}/rollback/RollbackRequest.java | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 16 +-
.../apache/hudi/client/TestHoodieClientBase.java | 3 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 48 ++-
.../hudi/common/HoodieMergeOnReadTestUtils.java | 40 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 15 +-
.../java/org/apache/hudi/table/TestCleaner.java | 2 +-
.../apache/hudi/table/TestMergeOnReadTable.java | 287 ++++++++++---
.../commit/TestCopyOnWriteActionExecutor.java} | 214 ++++------
.../table/action/commit/TestUpsertPartitioner.java | 148 +++++++
hudi-common/pom.xml | 24 ++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 43 +-
.../org/apache/hudi/common/fs/StorageSchemes.java | 2 +
.../hudi/common/model/WriteOperationType.java | 4 +
.../table/timeline/HoodieDefaultTimeline.java | 9 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 5 +
.../table/timeline/TimelineMetadataUtils.java | 15 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 72 +++-
.../hudi/avro/TestHoodieAvroWriteSupport.java | 21 +-
.../common/fs/inline/TestInLineFileSystem.java | 2 +
.../hudi/common/storage/TestStorageSchemes.java | 1 +
.../table/timeline/TestHoodieActiveTimeline.java | 16 +-
hudi-hadoop-mr/pom.xml | 24 ++
.../org/apache/hudi/hadoop/HoodieHiveUtil.java | 24 ++
.../hudi/hadoop/HoodieParquetInputFormat.java | 67 ++-
.../realtime/AbstractRealtimeRecordReader.java | 4 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 7 +
.../apache/hudi/hadoop/InputFormatTestUtil.java | 7 +-
.../apache/hudi/hadoop/InputPathHandlerTest.java | 28 +-
.../org/apache/hudi/hadoop/TestAnnotation.java | 4 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 122 ++++++
.../hudi/hadoop/TestRecordReaderValueIterator.java | 13 +-
hudi-hive-sync/pom.xml | 26 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 297 +++++++------
.../test/java/org/apache/hudi/hive/TestUtil.java | 2 +-
hudi-integ-test/pom.xml | 24 +-
.../java/org/apache/hudi/integ/ITTestBase.java | 18 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 20 +-
.../org/apache/hudi/integ/ITTestHoodieSanity.java | 33 +-
hudi-spark/pom.xml | 29 +-
hudi-spark/src/test/java/DataSourceUtilsTest.java | 6 +-
hudi-spark/src/test/scala/TestDataSource.scala | 21 +-
.../src/test/scala/TestDataSourceDefaults.scala | 18 +-
hudi-timeline-service/pom.xml | 22 +-
hudi-utilities/pom.xml | 24 ++
.../org/apache/hudi/utilities/UtilHelpers.java | 28 +-
.../checkpointing/InitialCheckPointProvider.java | 33 +-
.../checkpointing/KafkaConnectHdfsProvider.java | 27 +-
.../deltastreamer/HoodieDeltaStreamer.java | 59 +--
.../HoodieMultiTableDeltaStreamer.java | 2 +-
.../hudi/utilities/TestHoodieDeltaStreamer.java | 34 +-
.../TestKafkaConnectHdfsProvider.java | 19 +-
.../transform/TestChainedTransformer.java | 14 +-
pom.xml | 34 +-
scripts/run_travis_tests.sh | 7 +
108 files changed, 5229 insertions(+), 2057 deletions(-)
create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java
copy hudi-utilities/src/test/resources/delta-streamer-config/base.properties => hudi-cli/src/test/resources/clean.properties (87%)
copy hudi-client/src/main/java/org/apache/hudi/exception/{HoodieRollbackException.java => HoodieRestoreException.java} (81%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/BucketInfo.java (60%)
copy docker/demo/presto-table-check.commands => hudi-client/src/main/java/org/apache/hudi/table/action/commit/BucketType.java (89%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertCommitActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertHelper.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertPreppedCommitActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/CommitActionExecutor.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/DeleteCommitActionExecutor.java} (55%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/DeleteHelper.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/HoodieWriteMetadata.java
copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/InsertBucket.java (58%)
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/InsertCommitActionExecutor.java} (50%)
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/InsertPreppedCommitActionExecutor.java} (54%)
copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java (61%)
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/UpsertCommitActionExecutor.java} (50%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/UpsertPreppedCommitActionExecutor.java} (54%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/WriteHelper.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/BulkInsertDeltaCommitActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/BulkInsertPreppedDeltaCommitActionExecutor.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/DeleteDeltaCommitActionExecutor.java} (51%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/DeltaCommitActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/InsertDeltaCommitActionExecutor.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/InsertPreppedDeltaCommitActionExecutor.java} (52%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/UpsertDeltaCommitActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/UpsertDeltaCommitPartitioner.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/UpsertPreppedDeltaCommitActionExecutor.java} (52%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/CopyOnWriteRestoreActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/MergeOnReadRestoreActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java
rename hudi-client/src/main/java/org/apache/hudi/table/{ => action}/rollback/RollbackHelper.java (99%)
rename hudi-client/src/main/java/org/apache/hudi/table/{ => action}/rollback/RollbackRequest.java (98%)
rename hudi-client/src/test/java/org/apache/hudi/table/{TestCopyOnWriteTable.java => action/commit/TestCopyOnWriteActionExecutor.java} (68%)
create mode 100644 hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java