You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by na...@apache.org on 2020/04/10 17:32:33 UTC
[incubator-hudi] branch hudi_test_suite_refactor updated (5ee8a85
-> d2d2866)
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
discard 5ee8a85 Build fixes after rebase
discard 3e2e710 Fix Compilation Issues + Port Bug Fixes
omit 29b4fdf [HUDI-394] Provide a basic implementation of test suite
add 6808559 [HUDI-717] Fixed usage of HiveDriver for DDL statements. (#1416)
add deb95ad [HUDI-748] Adding .codecov.yml to set exclusions for code coverage reports. (#1468)
add 575d87c HUDI-644 kafka connect checkpoint provider (#1453)
add eaf6cc2 [HUDI-756] Organize Cleaning Action execution into a single package in hudi-client (#1485)
add b5d093a [MINOR] Clear up the redundant comment. (#1489)
add d610252 [HUDI-288]: Add support for ingesting multiple kafka streams in a single DeltaStreamer deployment (#1150)
add 4e5c867 [HUDI-740]Fix can not specify the sparkMaster and code clean for SparkUtil (#1452)
add f7b55af [MINOR] Fix typo in TimelineService (#1497)
add 1f6be82 [HUDI-758] Modify Integration test to include incremental queries for MOR tables
add 3c80342 rename variable per review comments
add 996f761 Trying git merge --squash
add f5f34bb [HUDI-568] Improve unit test coverage
add 8c7cef3 [HUDI - 738] Add validation to DeltaStreamer to fail fast when filterDupes is enabled on UPSERT mode. (#1505)
add 1450004 [HUDI-394] Provide a basic implementation of test suite
add ab98c40 Fix Compilation Issues + Port Bug Fixes
add b519057 Build fixes after rebase
add d2d2866 more fixes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5ee8a85)
\
N -- N -- N refs/heads/hudi_test_suite_refactor (d2d2866)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.codecov.yml | 44 +++
...n_commit_time.sh => get_min_commit_time_cow.sh} | 0
...n_commit_time.sh => get_min_commit_time_mor.sh} | 2 +-
...ntal.commands => hive-incremental-cow.commands} | 2 +-
...n.commands => hive-incremental-mor-ro.commands} | 9 +-
...n.commands => hive-incremental-mor-rt.commands} | 9 +-
.../org/apache/hudi/cli/HoodieCliSparkConfig.java | 46 +++
.../apache/hudi/cli/commands/CleansCommand.java | 2 +-
.../hudi/cli/commands/CompactionCommand.java | 16 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 62 ++--
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 36 +-
.../apache/hudi/client/AbstractHoodieClient.java | 4 -
.../hudi/client/AbstractHoodieWriteClient.java | 16 +-
.../org/apache/hudi/client/HoodieCleanClient.java | 197 ----------
.../org/apache/hudi/client/HoodieReadClient.java | 2 +-
.../org/apache/hudi/client/HoodieWriteClient.java | 62 ++--
.../apache/hudi/table/HoodieCommitArchiveLog.java | 12 +-
.../apache/hudi/table/HoodieCopyOnWriteTable.java | 172 +--------
.../apache/hudi/table/HoodieMergeOnReadTable.java | 5 +-
.../java/org/apache/hudi/table/HoodieTable.java | 57 ++-
.../action/BaseActionExecutor.java} | 25 +-
.../table/action/clean/CleanActionExecutor.java | 280 +++++++++++++++
.../clean/CleanPlanner.java} | 11 +-
.../table/action/clean/PartitionCleanStat.java | 70 ++++
.../compact/HoodieMergeOnReadTableCompactor.java | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 6 +-
.../apache/hudi/client/TestHoodieClientBase.java | 11 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 2 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 6 +-
.../hudi/common/HoodieTestDataGenerator.java | 157 ++++++--
.../java/org/apache/hudi/index/TestHbaseIndex.java | 26 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 16 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 8 +-
.../io/storage/TestHoodieStorageWriterFactory.java | 2 +-
.../java/org/apache/hudi/table/TestCleaner.java | 48 ++-
.../apache/hudi/table/TestCopyOnWriteTable.java | 20 +-
.../apache/hudi/table/TestMergeOnReadTable.java | 48 ++-
.../hudi/table/compact/TestAsyncCompaction.java | 2 +-
.../hudi/table/compact/TestHoodieCompactor.java | 10 +-
.../org/apache/hudi/common/model/HoodieKey.java | 3 +-
.../org/apache/hudi/common/util/CleanerUtils.java | 5 +-
.../hudi/common/util/collection/RocksDBDAO.java | 3 -
.../hudi/avro/TestHoodieAvroWriteSupport.java | 69 ++++
.../apache/hudi/common/model/HoodieTestUtils.java | 2 +-
.../common/table/TestHoodieTableMetaClient.java | 19 +
.../table/view/TestIncrementalFSViewSync.java | 3 +-
.../apache/hudi/common/util/TestParquetUtils.java | 2 +-
.../common/util/collection/TestRocksDBManager.java | 168 +++++++--
hudi-hadoop-mr/pom.xml | 6 +
.../realtime/TestHoodieRealtimeFileSplit.java | 162 +++++++++
.../realtime/TestHoodieRealtimeRecordReader.java | 10 +-
.../org/apache/hudi/hive/HoodieHiveClient.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 97 ++++-
.../test/java/org/apache/hudi/hive/TestUtil.java | 25 +-
.../org/apache/hudi/hive/util/HiveTestService.java | 30 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 53 ++-
.../main/java/org/apache/hudi/QuickstartUtils.java | 9 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 1 -
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 17 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 1 +
.../org/apache/hudi/IncrementalRelation.scala | 2 +-
.../hudi/timeline/service/TimelineService.java | 2 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 53 +--
.../InitialCheckPointProvider.java} | 19 +-
.../checkpointing/KafkaConnectHdfsProvider.java | 152 ++++++++
.../hudi/utilities/deltastreamer/DeltaSync.java | 14 +-
.../deltastreamer/HoodieDeltaStreamer.java | 87 +++--
.../HoodieMultiTableDeltaStreamer.java | 396 +++++++++++++++++++++
.../deltastreamer/SourceFormatAdapter.java | 8 +-
.../deltastreamer/TableExecutionContext.java | 85 +++++
.../hudi/utilities/TestHoodieDeltaStreamer.java | 126 +++++--
.../TestHoodieMultiTableDeltaStreamer.java | 166 +++++++++
.../apache/hudi/utilities/UtilitiesTestBase.java | 69 ++--
.../TestKafkaConnectHdfsProvider.java | 94 +++++
.../utilities/sources/AbstractBaseTestSource.java | 19 +-
.../sources/AbstractDFSSourceTestBase.java | 2 +-
.../hudi/utilities/sources/TestKafkaSource.java | 9 +-
.../utilities/sources/TestParquetDFSSource.java | 2 +-
...es => invalid_hive_sync_uber_config.properties} | 6 +-
...roperties => short_trip_uber_config.properties} | 7 +-
...-flattened.avsc => source_short_trip_uber.avsc} | 21 +-
.../{source-flattened.avsc => source_uber.avsc} | 21 +-
...-flattened.avsc => target_short_trip_uber.avsc} | 21 +-
.../{source-flattened.avsc => target_uber.avsc} | 21 +-
...ansformer.properties => uber_config.properties} | 8 +-
86 files changed, 2619 insertions(+), 989 deletions(-)
create mode 100644 .codecov.yml
copy docker/demo/{get_min_commit_time.sh => get_min_commit_time_cow.sh} (100%)
rename docker/demo/{get_min_commit_time.sh => get_min_commit_time_mor.sh} (90%)
rename docker/demo/{hive-incremental.commands => hive-incremental-cow.commands} (93%)
copy docker/demo/{hive-batch2-after-compaction.commands => hive-incremental-mor-ro.commands} (75%)
copy docker/demo/{hive-batch2-after-compaction.commands => hive-incremental-mor-rt.commands} (75%)
create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCliSparkConfig.java
delete mode 100644 hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieIOHandle.java => table/action/BaseActionExecutor.java} (68%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
rename hudi-client/src/main/java/org/apache/hudi/table/{CleanHelper.java => action/clean/CleanPlanner.java} (97%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/clean/PartitionCleanStat.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java
create mode 100644 hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeFileSplit.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/{exception/HoodieDeltaStreamerException.java => checkpointing/InitialCheckPointProvider.java} (75%)
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/checkpointing/KafkaConnectHdfsProvider.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/TableExecutionContext.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMultiTableDeltaStreamer.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java
copy hudi-utilities/src/test/resources/delta-streamer-config/{sql-transformer.properties => invalid_hive_sync_uber_config.properties} (73%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{sql-transformer.properties => short_trip_uber_config.properties} (69%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source-flattened.avsc => source_short_trip_uber.avsc} (79%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source-flattened.avsc => source_uber.avsc} (79%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source-flattened.avsc => target_short_trip_uber.avsc} (79%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source-flattened.avsc => target_uber.avsc} (79%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{sql-transformer.properties => uber_config.properties} (67%)