You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/03/07 14:54:49 UTC
(hudi) branch master updated: [HUDI-5167] Reducing total test run time: reducing tests for virtual keys (#7153)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d8e675d035a [HUDI-5167] Reducing total test run time: reducing tests for virtual keys (#7153)
d8e675d035a is described below
commit d8e675d035aa7e77c7e476d2c132c56979948e38
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Thu Mar 7 06:54:43 2024 -0800
[HUDI-5167] Reducing total test run time: reducing tests for virtual keys (#7153)
---
.../TestHoodieClientOnCopyOnWriteStorage.java | 111 +++++++++------------
1 file changed, 49 insertions(+), 62 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 44105a41983..61f73207aeb 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -204,7 +204,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private static Stream<Arguments> rollbackFailedCommitsParams() {
return Stream.of(
Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, true),
- Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, false),
Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, true),
Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, false)
);
@@ -240,10 +239,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test Auto Commit behavior for HoodieWriteClient insertPrepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testAutoCommitOnInsertPrepped(boolean populateMetaFields) throws Exception {
- testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true, populateMetaFields);
+ @Test
+ public void testAutoCommitOnInsertPrepped() throws Exception {
+ testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true, true);
}
/**
@@ -276,11 +274,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test Auto Commit behavior for HoodieWriteClient bulk-insert prepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testAutoCommitOnBulkInsertPrepped(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testAutoCommitOnBulkInsertPrepped() throws Exception {
testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsertPreppedRecords(recordRDD, instantTime,
- Option.empty()), true, populateMetaFields);
+ Option.empty()), true, true);
}
/**
@@ -440,10 +437,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test De-duplication behavior for HoodieWriteClient upsert API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeduplicationOnUpsert(boolean populateMetaFields) throws Exception {
- testDeduplication(SparkRDDWriteClient::upsert, populateMetaFields);
+ @Test
+ public void testDeduplicationOnUpsert() throws Exception {
+ testDeduplication(SparkRDDWriteClient::upsert, true);
}
/**
@@ -598,11 +594,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test UpsertPrepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testUpsertsPrepped(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testUpsertsPrepped() throws Exception {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
testUpsertsInternal(cfgBuilder.build(), SparkRDDWriteClient::upsertPreppedRecords, true);
}
@@ -837,11 +832,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test InsertPrepped API for HoodieConcatHandle.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testInsertsPreppedWithHoodieConcatHandle(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
testHoodieConcatHandle(cfgBuilder.build(), true);
}
@@ -994,11 +988,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Tests deletion of records.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeletes(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testDeletes() throws Exception {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
/**
* Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
@@ -1019,7 +1012,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
// unused as genFn uses hard-coded number of inserts/updates/deletes
-1, recordGenFunction, SparkRDDWriteClient::upsert, true, 200, 200, 1, false,
- populateMetaFields);
+ true);
/**
* Write 2 (deletes+writes).
@@ -1037,7 +1030,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
};
writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 100, recordGenFunction,
SparkRDDWriteClient::upsert, true, 50, 150, 2, false,
- populateMetaFields);
+ true);
}
/**
@@ -1046,11 +1039,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*
* @throws Exception
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeletesForInsertsInSameBatch(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testDeletesForInsertsInSameBatch() throws Exception {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
/**
* Write 200 inserts and issue deletes to a subset(50) of inserts.
@@ -1071,7 +1063,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
-1, recordGenFunction, SparkRDDWriteClient::upsert, true, 150, 150, 1, false,
- populateMetaFields);
+ true);
}
private void assertPartitionPathRecordKeys(List<Pair<String, String>> expectedPartitionPathRecKeyPairs, String[] fullPartitionPaths) {
@@ -1898,19 +1890,17 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test scenario of writing fewer file groups than existing number of file groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testInsertOverwritePartitionHandlingWithFewerRecords(boolean populateMetaFields) throws Exception {
- verifyInsertOverwritePartitionHandling(3000, 1000, populateMetaFields);
+ @Test
+ public void testInsertOverwritePartitionHandlingWithFewerRecords() throws Exception {
+ verifyInsertOverwritePartitionHandling(3000, 1000, true);
}
/**
* Test scenario of writing similar number file groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testInsertOverwritePartitionHandlingWithSimilarNumberOfRecords(boolean populateMetaFields) throws Exception {
- verifyInsertOverwritePartitionHandling(3000, 3000, populateMetaFields);
+ @Test
+ public void testInsertOverwritePartitionHandlingWithSimilarNumberOfRecords() throws Exception {
+ verifyInsertOverwritePartitionHandling(3000, 3000, true);
}
/**
@@ -1963,19 +1953,17 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test scenario of writing similar number file groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void verifyDeletePartitionsHandlingWithSimilarNumberOfRecords(boolean populateMetaFields) throws Exception {
- verifyDeletePartitionsHandling(3000, 3000, 3000, populateMetaFields);
+ @Test
+ public void verifyDeletePartitionsHandlingWithSimilarNumberOfRecords() throws Exception {
+ verifyDeletePartitionsHandling(3000, 3000, 3000, true);
}
/**
* Test scenario of writing more file groups for first partition than second and third partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void verifyDeletePartitionsHandlingHandlingWithFewerRecordsSecondThirdPartition(boolean populateMetaFields) throws Exception {
- verifyDeletePartitionsHandling(3000, 1000, 1000, populateMetaFields);
+ @Test
+ public void verifyDeletePartitionsHandlingHandlingWithFewerRecordsSecondThirdPartition() throws Exception {
+ verifyDeletePartitionsHandling(3000, 1000, 1000, true);
}
private Set<String> insertPartitionRecordsWithCommit(SparkRDDWriteClient client, int recordsCount, String commitTime1, String partitionPath) throws IOException {
@@ -2217,11 +2205,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test to ensure commit metadata points to valid files.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testCommitWritesRelativePaths(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testCommitWritesRelativePaths() throws Exception {
+
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
HoodieSparkTable table = HoodieSparkTable.create(cfgBuilder.build(), context, metaClient);
@@ -2393,9 +2381,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
}
@ParameterizedTest
- @MethodSource("rollbackAfterConsistencyCheckFailureParams")
- public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean enableOptimisticConsistencyGuard, boolean populateMetCols) throws Exception {
- testRollbackAfterConsistencyCheckFailureUsingFileList(false, enableOptimisticConsistencyGuard, populateMetCols);
+ @ValueSource(booleans = {true, false})
+ public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean enableOptimisticConsistencyGuard) throws Exception {
+ testRollbackAfterConsistencyCheckFailureUsingFileList(false, enableOptimisticConsistencyGuard, true);
}
@ParameterizedTest
@@ -2486,9 +2474,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
}
}
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testRollbackFailedCommitsToggleCleaningPolicy(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception {
+ boolean populateMetaFields = true;
HoodieTestUtils.init(hadoopConf, basePath);
HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
@@ -2655,12 +2643,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
return Pair.of(markerFilePath.get(), result);
}
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testMultiOperationsPerCommit(boolean populateMetaFields) throws IOException {
+ @Test
+ public void testMultiOperationsPerCommit() throws IOException {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false)
.withAllowMultiWriteOnSameInstant(true);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
HoodieWriteConfig cfg = cfgBuilder.build();
SparkRDDWriteClient client = getHoodieWriteClient(cfg);
String firstInstantTime = "0000";