You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2024/06/04 09:21:44 UTC
(hudi) branch master updated: [HUDI-7704] Unify test client storage classes with duplicate code (#11152)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 06457c0aff2 [HUDI-7704] Unify test client storage classes with duplicate code (#11152)
06457c0aff2 is described below
commit 06457c0aff2a102a0c760e60158280004914c0ca
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Tue Jun 4 16:21:37 2024 +0700
[HUDI-7704] Unify test client storage classes with duplicate code (#11152)
Co-authored-by: Vova Kolmakov <ko...@huawei-partners.com>
---
.../java/org/apache/hudi/testutils/Assertions.java | 97 ++
.../hudi/utils/HoodieWriterClientTestHarness.java | 1297 +++++++++++++-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 21 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 1421 ++-------------
.../testutils/HoodieJavaClientTestHarness.java | 31 +-
.../hudi/client/TestTableSchemaEvolution.java | 2 +-
.../functional/TestHoodieBackedMetadata.java | 21 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 1824 +++-----------------
.../hudi/testutils/HoodieClientTestBase.java | 3 +
.../testutils/HoodieSparkClientTestHarness.java | 29 +-
.../quickstart/TestHoodieFlinkQuickstart.java | 1 -
.../common/testutils/HoodieCommonTestHarness.java | 6 +
.../apache/hudi/TestDataSourceReadWithDeletes.java | 3 +-
13 files changed, 1780 insertions(+), 2976 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
index bb2ba84f8c9..f2d78c1eaec 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
@@ -20,13 +20,28 @@
package org.apache.hudi.testutils;
import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodiePartitionMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordDelegate;
import org.apache.hudi.common.testutils.CheckedFunction;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.RawTripTestPayload;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Commonly used assertion functions.
@@ -51,4 +66,86 @@ public class Assertions {
assertEquals(fileSizeGetter.apply(status), status.getStat().getFileSizeInBytes())));
}
+ public static void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords,
+ HoodieStorage storage) throws IOException {
+ String[] partitionPathSet = inputRecords.stream()
+ .map(HoodieRecord::getPartitionPath).distinct().toArray(String[]::new);
+ assertPartitionMetadata(basePath, partitionPathSet, storage);
+ }
+
+ public static void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys,
+ HoodieStorage storage) throws IOException {
+ String[] partitionPathSet = inputKeys.stream()
+ .map(HoodieKey::getPartitionPath).distinct().toArray(String[]::new);
+ assertPartitionMetadata(basePath, partitionPathSet, storage);
+ }
+
+ /**
+ * Ensure presence of partition meta-data at known depth.
+ *
+ * @param partitionPaths Partition paths to check
+ * @param storage {@link HoodieStorage} instance.
+ * @throws IOException in case of error
+ */
+ public static void assertPartitionMetadata(String basePath, String[] partitionPaths,
+ HoodieStorage storage) throws IOException {
+ for (String partitionPath : partitionPaths) {
+ assertTrue(
+ HoodiePartitionMetadata.hasPartitionMetadata(
+ storage, new StoragePath(basePath, partitionPath)));
+ HoodiePartitionMetadata pmeta =
+ new HoodiePartitionMetadata(storage, new StoragePath(basePath, partitionPath));
+ pmeta.readFromFS();
+ assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth());
+ }
+ }
+
+ /**
+ * Assert that there is no duplicate key at the partition level.
+ *
+ * @param records List of Hoodie records
+ */
+ public static void assertNoDupesWithinPartition(List<HoodieRecord<RawTripTestPayload>> records) {
+ Map<String, Set<String>> partitionToKeys = new HashMap<>();
+ for (HoodieRecord r : records) {
+ String key = r.getRecordKey();
+ String partitionPath = r.getPartitionPath();
+ if (!partitionToKeys.containsKey(partitionPath)) {
+ partitionToKeys.put(partitionPath, new HashSet<>());
+ }
+ assertFalse(partitionToKeys.get(partitionPath).contains(key), "key " + key + " is duplicate within partition " + partitionPath);
+ partitionToKeys.get(partitionPath).add(key);
+ }
+ }
+
+ /**
+ * Assert that there is no duplicate key at the partition level.
+ *
+ * @param recordDelegates List of Hoodie record delegates
+ */
+ public static void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
+ Map<String, Set<String>> partitionToKeys = new HashMap<>();
+ for (HoodieRecordDelegate r : recordDelegates) {
+ String recordKey = r.getRecordKey();
+ String partitionPath = r.getPartitionPath();
+ if (!partitionToKeys.containsKey(partitionPath)) {
+ partitionToKeys.put(partitionPath, new HashSet<>());
+ }
+ assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
+ partitionToKeys.get(partitionPath).add(recordKey);
+ }
+ }
+
+ public static void assertActualAndExpectedPartitionPathRecordKeyMatches(List<Pair<String, String>> expectedPartitionPathRecKeyPairs,
+ List<Pair<String, String>> actualPartitionPathRecKeyPairs) {
+ // verify all partitionpath, record key matches
+ assertEquals(expectedPartitionPathRecKeyPairs.size(), actualPartitionPathRecKeyPairs.size());
+ for (Pair<String, String> entry : actualPartitionPathRecKeyPairs) {
+ assertTrue(expectedPartitionPathRecKeyPairs.contains(entry));
+ }
+
+ for (Pair<String, String> entry : expectedPartitionPathRecKeyPairs) {
+ assertTrue(actualPartitionPathRecKeyPairs.contains(entry));
+ }
+ }
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
index 2d4e87c52e6..0d2944bf408 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
@@ -18,44 +18,126 @@
package org.apache.hudi.utils;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.client.BaseHoodieWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.config.LockConfiguration;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieAvroRecord;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
+import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodiePartitionMetadata;
+import org.apache.hudi.common.model.HoodieOperation;
+import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.marker.MarkerType;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.RawTripTestPayload;
+import org.apache.hudi.common.util.ClusteringUtils;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.FileFormatUtils;
+import org.apache.hudi.common.util.MarkerUtils;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCleanConfig;
+import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieLockConfig;
+import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieCommitException;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.keygen.KeyGenerator;
+import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.HoodieWriteMetadata;
+import org.apache.hudi.table.action.commit.HoodieWriteHelper;
+import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.hudi.testutils.MetadataMergeWriteStatus;
+import org.jetbrains.annotations.NotNull;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
+import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
import static org.apache.hudi.common.testutils.HoodieTestUtils.RAW_TRIPS_TEST_NAME;
+import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
+import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
+import static org.apache.hudi.config.HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING;
+import static org.apache.hudi.testutils.Assertions.assertNoDupesWithinPartition;
+import static org.apache.hudi.testutils.Assertions.assertNoDuplicatesInPartition;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
-public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
+public abstract class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
protected static int timelineServicePort = FileSystemViewStorageConfig.REMOTE_PORT_NUM.defaultValue();
+ protected static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
+
+ protected HoodieTestTable testTable;
+
+ protected abstract BaseHoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg);
protected void addConfigsForPopulateMetaFields(HoodieWriteConfig.Builder configBuilder, boolean populateMetaFields,
boolean isMetadataTable) {
@@ -76,8 +158,8 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
public static Properties getPropertiesForKeyGen(boolean populateMetaFields) {
Properties properties = new Properties();
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields));
- properties.put("hoodie.datasource.write.recordkey.field", "_row_key");
- properties.put("hoodie.datasource.write.partitionpath.field", "partition_path");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key");
properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path");
return properties;
@@ -86,7 +168,7 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
protected Properties getPropertiesForMetadataTable() {
Properties properties = new Properties();
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false");
- properties.put("hoodie.datasource.write.recordkey.field", "key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "key");
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "key");
return properties;
}
@@ -165,71 +247,1182 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
return builder;
}
- public void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords,
- HoodieStorage storage) throws IOException {
- Set<String> partitionPathSet = inputRecords.stream()
- .map(HoodieRecord::getPartitionPath)
- .collect(Collectors.toSet());
- assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
+ // Functional Interfaces for passing lambda and Hoodie Write API contexts
+
+ @FunctionalInterface
+ public interface Function2<R, T1, T2> {
+
+ R apply(T1 v1, T2 v2) throws IOException;
+ }
+
+ @FunctionalInterface
+ public interface Function3<R, T1, T2, T3> {
+
+ R apply(T1 v1, T2 v2, T3 v3) throws IOException;
+ }
+
+ /* Auxiliary methods for testing CopyOnWriteStorage with Spark and Java clients
+ to avoid code duplication in TestHoodieClientOnCopyOnWriteStorage and TestHoodieJavaClientOnCopyOnWriteStorage */
+
+ protected abstract List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff)
+ throws IOException;
+
+ protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit) throws Exception {
+ return castInsertFirstBatch(writeConfig, client, newCommitTime, initCommitTime, numRecordsInThisCommit, writeFn,
+ isPreppedAPI, assertForCommit, expRecordsInThisCommit, true);
+ }
+
+ protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+ return null; // override in subclasses if needed
+ }
+
+ protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit) throws Exception {
+ return castWriteBatch(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+ writeFn, assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, true);
+ }
+
+ protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ return null; // override in subclasses if needed
+ }
+
+ protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ return null; // override in subclasses if needed
+ }
+
+ protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, boolean assertForCommit,
+ int expRecordsInThisCommit, int expTotalRecords) throws Exception {
+ return castDeleteBatch(writeConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit, isPreppedAPI,
+ assertForCommit, expRecordsInThisCommit, expTotalRecords, true);
+ }
+
+ protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+ return null; // override in subclasses if needed
+ }
+
+ protected Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI, HoodieWriteConfig writeConfig,
+ Function2<List<HoodieRecord>, String, Integer> wrapped) {
+ return null; // override in subclasses if needed
+ }
+
+ protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+ return new String[0]; // override in subclasses if needed
+ }
+
+ protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
+ // override in subclasses if needed
+ }
+
+ protected HoodieWriteConfig getRollbackMarkersAndConsistencyGuardWriteConfig(
+ boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard, boolean populateMetaFields) {
+ Properties properties = new Properties();
+ if (!populateMetaFields) {
+ properties = getPropertiesForKeyGen();
+ }
+
+ return !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+ .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1)
+ .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
+ .build() :
+ getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
+ .withConsistencyCheckEnabled(true)
+ .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
+ .withOptimisticConsistencyGuardSleepTimeMs(1).build())
+ .withProperties(properties).build();
+ }
+
+ protected HoodieWriteConfig getConsistencyCheckWriteConfig(boolean enableOptimisticConsistencyGuard) {
+ return !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+ .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
+ .build()) : (getConfigBuilder().withAutoCommit(false)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+ .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
+ .withOptimisticConsistencyGuardSleepTimeMs(1).build())
+ .build());
+ }
+
+ protected HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
+ Properties properties = new Properties();
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
+ if (!populateMetaFields) {
+ properties.putAll(getPropertiesForKeyGen(populateMetaFields));
+ }
+ return getConfigBuilder()
+ .withEmbeddedTimelineServerEnabled(false)
+ .withCleanConfig(HoodieCleanConfig.newBuilder()
+ .withFailedWritesCleaningPolicy(cleaningPolicy)
+ .withAutoClean(false).build())
+ .withTimelineLayoutVersion(1)
+ .withHeartbeatIntervalInMs(3 * 1000)
+ .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+ .withRemoteServerPort(timelineServicePort).build())
+ .withAutoCommit(false)
+ .withLockConfig(HoodieLockConfig.newBuilder()
+ .withLockProvider(InProcessLockProvider.class)
+ .build())
+ .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
+ .withProperties(properties).build();
+ }
+
+ @NotNull
+ protected Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
+ for (WriteStatus status : allStatus) {
+ StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
+ records.addAll(getFileUtilsInstance(createMetaClient()).readAvroRecords(storage, filePath));
+ }
+ Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
+ assertEquals(records.size(), expectedKeys.size());
+ return expectedKeys;
+ }
+
+ protected void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
+ List<GenericRecord> records = new ArrayList<>();
+ Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
+ Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
+ .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
+ assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
+ Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
+ for (GenericRecord record : records) {
+ String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+ assertTrue(expectedKeys.contains(recordKey));
+ String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
+ assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
+ }
+ }
+
+ protected Pair<String, String> getPartitionAndBaseFilePathsFromLatestCommitMetadata(HoodieTableMetaClient metaClient) throws IOException {
+ String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+ .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
+ metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
+ HoodieCommitMetadata.class);
+ String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
+ .flatMap(Collection::stream).filter(s -> s.getPath().endsWith(extension)).findAny()
+ .map(HoodieWriteStat::getPath).orElse(null);
+ String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
+ .flatMap(Collection::stream).filter(s -> s.getPath().endsWith(extension)).findAny()
+ .map(HoodieWriteStat::getPartitionPath).orElse(null);
+ return Pair.of(partitionPath, metaClient.getBasePathV2().toString() + StoragePath.SEPARATOR + filePath);
}
- public void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys,
- HoodieStorage storage) throws IOException {
- Set<String> partitionPathSet = inputKeys.stream()
- .map(HoodieKey::getPartitionPath)
- .collect(Collectors.toSet());
- assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
+ public static FileFormatUtils getFileUtilsInstance(HoodieTableMetaClient metaClient) {
+ return HoodieIOFactory.getIOFactory(metaClient.getStorage())
+ .getFileFormatUtils(metaClient.getTableConfig().getBaseFileFormat());
}
/**
- * Ensure presence of partition meta-data at known depth.
- *
- * @param partitionPaths Partition paths to check
- * @param storage {@link HoodieStorage} instance.
- * @throws IOException in case of error
+ * Build Hoodie Write Config for small data file sizes.
+ */
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, boolean mergeAllowDuplicateInserts) {
+ return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, dataGen.getEstimatedFileSizeInBytes(150), mergeAllowDuplicateInserts);
+ }
+
+ /**
+ * Build Hoodie Write Config for specified small file sizes.
*/
- public static void assertPartitionMetadata(String basePath, String[] partitionPaths,
- HoodieStorage storage) throws IOException {
- for (String partitionPath : partitionPaths) {
- assertTrue(
- HoodiePartitionMetadata.hasPartitionMetadata(
- storage, new StoragePath(basePath, partitionPath)));
- HoodiePartitionMetadata pmeta =
- new HoodiePartitionMetadata(storage, new StoragePath(basePath, partitionPath));
- pmeta.readFromFS();
- assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth());
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+ String schemaStr = useNullSchema ? NULL_SCHEMA : TRIP_EXAMPLE_SCHEMA;
+ return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts);
+ }
+
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize) {
+ return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false);
+ }
+
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+ return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts, true, new Properties());
+ }
+
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
+ return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
+ }
+
+ protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
+ boolean populateMetaFields, Properties props) {
+ HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
+ if (!populateMetaFields) {
+ builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.SIMPLE).build());
}
+ return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ .compactionSmallFileSize(smallFileSize)
+ // Set rollback to LAZY so no inflights are deleted
+ .insertSplitSize(insertSplitSize).build())
+ .withCleanConfig(HoodieCleanConfig.newBuilder()
+ .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+ .build())
+ .withStorageConfig(HoodieStorageConfig.newBuilder()
+ .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
+ .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
+ .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
+ .withProps(props)
+ .build();
}
- // Functional Interfaces for passing lambda and Hoodie Write API contexts
+ protected HoodieWriteConfig getSmallInsertWriteConfigForMDT(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+ HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
+ return builder.withCompactionConfig(
+ HoodieCompactionConfig.newBuilder()
+ .compactionSmallFileSize(smallFileSize)
+ // Set rollback to LAZY so no inflights are deleted
+ .insertSplitSize(insertSplitSize).build())
+ .withCleanConfig(HoodieCleanConfig.newBuilder()
+ .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
+ .withStorageConfig(
+ HoodieStorageConfig.newBuilder()
+ .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
+ .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
+ .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
+ .build();
+ }
- @FunctionalInterface
- public interface Function2<R, T1, T2> {
+ protected HoodieWriteConfig getWriteConfigWithPopulateMetaFieldsAndAllowOperationMetaField(boolean populateMetaFields, boolean allowOperationMetadataField) {
+ HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
+ .withAllowOperationMetadataField(allowOperationMetadataField).combineInput(true, true);
+ addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
+ return configBuilder.build();
+ }
- R apply(T1 v1, T2 v2) throws IOException;
+ protected List<HoodieRecord<RawTripTestPayload>> dedupForCopyOnWriteStorage(HoodieWriteConfig writeConfig, HoodieData<HoodieRecord> records,
+ boolean isGlobal, int additionalParallelism, int expectedNumPartitions) {
+ HoodieIndex index = mock(HoodieIndex.class);
+ when(index.isGlobal()).thenReturn(isGlobal);
+ int dedupParallelism = records.getNumPartitions() + additionalParallelism;
+ HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd = (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
+ .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
+ assertEquals(expectedNumPartitions, dedupedRecsRdd.getNumPartitions());
+ List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
+ assertEquals(isGlobal ? 1 : 2, dedupedRecs.size());
+ assertNoDupesWithinPartition(dedupedRecs);
+ return dedupedRecs;
}
- @FunctionalInterface
- public interface Function3<R, T1, T2, T3> {
+ protected HoodieTableMetaClient createMetaClient() {
+ return HoodieTestUtils.createMetaClient(storageConf, basePath);
+ }
- R apply(T1 v1, T2 v2, T3 v3) throws IOException;
+ protected HoodieTableMetaClient createMetaClient(String basePath) {
+ return HoodieTestUtils.createMetaClient(storageConf, basePath);
+ }
+
+ protected Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
+ return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
}
/**
- * Assert that there is no duplicate key at the partition level.
+ * Find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
*
- * @param records List of Hoodie records
- */
- public static void assertNodupesWithinPartition(List<HoodieRecord<RawTripTestPayload>> records) {
- Map<String, Set<String>> partitionToKeys = new HashMap<>();
- for (HoodieRecord r : records) {
- String key = r.getRecordKey();
- String partitionPath = r.getPartitionPath();
- if (!partitionToKeys.containsKey(partitionPath)) {
- partitionToKeys.put(partitionPath, new HashSet<>());
+ * @param partitionPath where parquet files created as part of clustering
+ * @throws IOException
+ */
+ protected void verifyClusteredFilesWithReplaceCommitMetadata(String partitionPath) throws IOException {
+ metaClient = HoodieTableMetaClient.reload(createMetaClient());
+ HoodieInstant replaceCommitInstant =
+ metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
+ HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
+ .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
+ HoodieReplaceCommitMetadata.class);
+
+ List<String> filesFromReplaceCommit = new ArrayList<>();
+ replaceCommitMetadata.getPartitionToWriteStats()
+ .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
+
+ // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
+ List<StoragePathInfo> pathInfoList =
+ storage.listDirectEntries(new StoragePath(basePath, partitionPath));
+ List<String> clusteredFiles = pathInfoList.stream()
+ .filter(entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
+ .map(pathInfo -> partitionPath + StoragePath.SEPARATOR + pathInfo.getPath().getName())
+ .collect(Collectors.toList());
+ assertEquals(clusteredFiles, filesFromReplaceCommit);
+ }
+
+ protected static List<Pair<HoodieInstant, HoodieClusteringPlan>> getAndAssertPendingClusteringPlans(boolean scheduleInlineClustering, HoodieTableMetaClient metaClient) {
+ List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
+ ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
+ if (scheduleInlineClustering) {
+ assertEquals(1, pendingClusteringPlans.size());
+ } else {
+ assertEquals(0, pendingClusteringPlans.size());
+ }
+ return pendingClusteringPlans;
+ }
+
+ protected void checkTimelineForUpsertsInternal(HoodieTableMetaClient metaClient) {
+ HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
+ List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
+ assertEquals(5, instants.size());
+ assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"), instants.get(0));
+ assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"), instants.get(1));
+ // New Format should have all states of instants
+ assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"), instants.get(2));
+ assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"), instants.get(3));
+ assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"), instants.get(4));
+ }
+
+ /**
+ * Verify data in base files matches expected records and commit time.
+ */
+ protected void verifyRecordsWritten(String commitTime, boolean populateMetadataField, List<HoodieRecord> expectedRecords,
+ List<WriteStatus> allStatus, HoodieWriteConfig config, KeyGenerator keyGenerator) {
+ List<GenericRecord> records = new ArrayList<>();
+ Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
+ if (config.populateMetaFields()) {
+ for (GenericRecord record : records) {
+ String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+ assertEquals(commitTime,
+ record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
+ assertTrue(expectedKeys.contains(recordKey));
+ }
+ } else {
+ for (GenericRecord record : records) {
+ String recordKey = keyGenerator.getKey(record).getRecordKey();
+ if (!populateMetadataField) {
+ assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
+ }
+ assertTrue(expectedKeys.contains(recordKey));
+ }
+ }
+ }
+
+ protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
+ return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
+ }
+
+ /**
+ * This method returns following three items:
+ * 1. List of all HoodieRecord written in the two batches of insert.
+ * 2. Commit instants of the two batches.
+ * 3. List of new file group ids that were written in the two batches.
+ */
+ protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> insertTwoBatches(BaseHoodieWriteClient writeClient, BaseHoodieWriteClient brokenClusteringClient,
+ boolean populateMetaFields, String partitionPath, boolean failInlineClustering) throws IOException {
+ BaseHoodieWriteClient client;
+ if (failInlineClustering) {
+ if (null != writeClient) {
+ writeClient.close();
+ }
+ client = brokenClusteringClient;
+ } else {
+ client = writeClient;
+ }
+
+ dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
+ String commitTime1 = client.createNewInstantTime();
+ List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
+ List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
+ Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
+
+ String commitTime2 = client.createNewInstantTime();
+ List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
+ List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
+ client.close();
+ Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
+ Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
+ fileIdsUnion.addAll(fileIds2);
+ //verify new files are created for 2nd write
+ Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
+ fileIdIntersection.retainAll(fileIds2);
+ assertEquals(0, fileIdIntersection.size());
+ return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
+ }
+
+ protected void rollbackAndAssert(boolean enableOptimisticConsistencyGuard, String instantTime, HoodieTableMetaClient metaClient, BaseHoodieWriteClient client) throws IOException {
+ if (!enableOptimisticConsistencyGuard) {
+ // Rollback of this commit should succeed with FailSafeCG
+ client.rollback(instantTime);
+ assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present");
+ // Marker directory must be removed after rollback
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+ } else {
+ // if optimistic CG is enabled, commit should have succeeded.
+ assertTrue(testTable.commitExists(instantTime),
+ "With optimistic CG, first commit should succeed. commit file should be present");
+ // Marker directory must be removed after rollback
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+ client.rollback(instantTime);
+ assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present");
+ }
+ }
+
+ protected Pair<StoragePath, List<WriteStatus>> testConsistencyCheck(HoodieEngineContext context, HoodieTableMetaClient metaClient,
+ String instantTime, boolean enableOptimisticConsistencyGuard,
+ Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+ Function transformInputFn, Function transformOutputFn)
+ throws Exception {
+ HoodieWriteConfig cfg = getConsistencyCheckWriteConfig(enableOptimisticConsistencyGuard);
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+
+ client.startCommitWithTime(instantTime);
+ List<HoodieRecord> writeRecords = dataGen.generateInserts(instantTime, 200);
+ List<WriteStatus> result = (List<WriteStatus>) transformOutputFn.apply(client.bulkInsert(transformInputFn.apply(writeRecords), instantTime));
+
+ // Create a dummy marker file to simulate the case that a marker file was created without data file.
+ // This should fail the commit
+ String partitionPath;
+ String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
+ if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
+ String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
+ markerFolderPath, storage, context, 1).values().stream()
+ .flatMap(Collection::stream).findFirst().get();
+ partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
+ } else {
+ partitionPath = storage.globEntries(
+ new StoragePath(String.format("%s/*/*/*/*", markerFolderPath)), path ->
+ path.toString().contains(HoodieTableMetaClient.MARKER_EXTN))
+ .stream()
+ .limit(1).map(status -> status.getPath().getParent().toString())
+ .collect(Collectors.toList()).get(0);
+ }
+
+ Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
+ cfg.getMarkersType(), getHoodieTableFn.apply(metaClient, cfg), instantTime)
+ .create(partitionPath,
+ FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(),
+ HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()),
+ IOType.MERGE);
+ if (!enableOptimisticConsistencyGuard) {
+ Exception e = assertThrows(HoodieCommitException.class, () -> client.commit(instantTime, transformInputFn.apply(result)),
+ "Commit should fail due to consistency check");
+ assertTrue(e.getCause() instanceof HoodieIOException);
+ } else {
+ // with optimistic CG, commit should succeed
+ client.commit(instantTime, transformInputFn.apply(result));
+ }
+ return Pair.of(markerFilePath.get(), result);
+ }
+
+ protected void testConsistencyCheckDuringFinalize(HoodieEngineContext context, boolean enableOptimisticConsistencyGuard,
+ Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+ Function transformInputFn, Function transformOutputFn) throws Exception {
+ HoodieTableMetaClient metaClient = createMetaClient();
+ String instantTime = "000";
+ HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
+ .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+ Pair<StoragePath, List<WriteStatus>> result = testConsistencyCheck(context, metaClient, instantTime,
+ enableOptimisticConsistencyGuard, getHoodieTableFn, transformInputFn, transformOutputFn);
+
+ // Delete orphan marker and commit should succeed
+ metaClient.getStorage().deleteFile(result.getKey());
+ if (!enableOptimisticConsistencyGuard) {
+ assertTrue(client.commit(instantTime, transformInputFn.apply(result.getRight())), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+ // Marker directory must be removed
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+ } else {
+ // with optimistic, first client.commit should have succeeded.
+ assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+ // Marker directory must be removed
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+ }
+ }
+
+ protected void testRollbackAfterConsistencyCheckFailureUsingFileList(HoodieEngineContext context, boolean rollbackUsingMarkers,
+ boolean enableOptimisticConsistencyGuard, boolean populateMetaFields,
+ Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+ Function transformInputFn, Function transformOutputFn) throws Exception {
+ String instantTime = "00000000000010";
+ HoodieTableMetaClient metaClient = createMetaClient();
+ HoodieWriteConfig cfg = getRollbackMarkersAndConsistencyGuardWriteConfig(rollbackUsingMarkers, enableOptimisticConsistencyGuard, populateMetaFields);
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+ testConsistencyCheck(context, metaClient, instantTime, enableOptimisticConsistencyGuard, getHoodieTableFn, transformInputFn, transformOutputFn);
+ rollbackAndAssert(enableOptimisticConsistencyGuard, instantTime, metaClient, client);
+ }
+
+ protected List<HoodieRecord> generate3AvroRecords(String commitTime) throws IOException {
+ String recordKey = UUID.randomUUID().toString();
+ HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
+ HoodieRecord<RawTripTestPayload> recordOne =
+ new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, commitTime), HoodieOperation.INSERT);
+ HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
+ HoodieRecord recordTwo =
+ new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, commitTime), HoodieOperation.INSERT);
+ // Same key and partition as keyTwo
+ HoodieRecord recordThree =
+ new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, commitTime), HoodieOperation.UPDATE_AFTER);
+ return Arrays.asList(recordOne, recordTwo, recordThree);
+ }
+
+ private HoodieWriteMetadata<List<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
+ boolean populateMetaFields,
+ boolean completeClustering,
+ String validatorClasses,
+ String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
+ Pair<List<HoodieRecord>, List<String>> allRecords,
+ Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> transformWriteMetadataFn,
+ Function<HoodieWriteConfig, KeyGenerator> createKeyGeneratorFn) {
+ HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
+ .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
+ .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
+ .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
+ .build();
+
+ HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
+ .withPreCommitValidatorConfig(validatorConfig)
+ .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
+ .withClusteringConfig(clusteringConfig).build();
+
+ // create client with new config.
+ BaseHoodieWriteClient client = getHoodieWriteClient(config);
+ String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
+ HoodieWriteMetadata<List<WriteStatus>> clusterMetadata = transformWriteMetadataFn.apply(client.cluster(clusteringCommitTime, completeClustering));
+ if (config.populateMetaFields()) {
+ verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses());
+ } else {
+ verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses(),
+ config, createKeyGeneratorFn.apply(config));
+ }
+ return clusterMetadata;
+ }
+
+ protected void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
+ String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
+ Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords,
+ Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> transformWriteMetadataFn,
+ Function<HoodieWriteConfig, KeyGenerator> createKeyGeneratorFn) {
+
+ HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
+ .withClusteringConfig(clusteringConfig)
+ .withProps(getPropertiesForKeyGen()).build();
+ HoodieWriteMetadata<List<WriteStatus>> clusterMetadata =
+ performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation,
+ sqlQueryForSingleResultValidation, allRecords.getLeft(), transformWriteMetadataFn, createKeyGeneratorFn);
+ if (assertSameFileIds) {
+ Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
+ .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
+ Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
+ assertEquals(insertedFileIds, replacedFileIds);
+ }
+ if (completeClustering) {
+ String clusteringCommitTime = createMetaClient().reloadActiveTimeline().getCompletedReplaceTimeline()
+ .getReverseOrderedInstants().findFirst().get().getTimestamp();
+ verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(),
+ clusterMetadata.getWriteStatuses(), config, createKeyGeneratorFn.apply(config));
+ }
+ }
+
+ protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, Function createBrokenClusteringClientFn) throws IOException {
+ return testInsertTwoBatches(populateMetaFields, "2015/03/16", createBrokenClusteringClientFn);
+ }
+
+ protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath,
+ Function createBrokenClusteringClientFn) throws IOException {
+ return testInsertTwoBatches(populateMetaFields, partitionPath, getPropertiesForKeyGen(populateMetaFields), false, createBrokenClusteringClientFn);
+ }
+
+ protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props, boolean failInlineClustering,
+ Function createBrokenClusteringClientFn) throws IOException {
+ // create config to not update small files.
+ HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
+ populateMetaFields ? props : getPropertiesForKeyGen());
+ return insertTwoBatches(getHoodieWriteClient(config), (BaseHoodieWriteClient) createBrokenClusteringClientFn.apply(config), populateMetaFields, partitionPath, failInlineClustering);
+ }
+
+ protected void testCommitWritesRelativePaths(Function transformInputFn) throws Exception {
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
+ try (BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
+ HoodieTableMetaClient metaClient = createMetaClient();
+ String instantTime = "000";
+ client.startCommitWithTime(instantTime);
+ List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
+
+ assertTrue(client.commit(instantTime, client.bulkInsert(transformInputFn.apply(records), instantTime)), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+
+ // Get base file paths from commit metadata
+ String actionType = metaClient.getCommitActionType();
+ HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
+ HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+ .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class);
+ String basePath = metaClient.getBasePathV2().toString();
+ Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
+
+ // Read from commit file
+ HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
+ metaClient.reloadActiveTimeline()
+ .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
+ HoodieCommitMetadata.class);
+ HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
+ // Compare values in both to make sure they are equal.
+ for (String pathName : paths.values()) {
+ assertTrue(commitPathNames.contains(pathName));
+ }
+ }
+ }
+
+ protected void testMetadataStatsOnCommit(boolean populateMetaFields, Function transformInputFn) throws Exception {
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
+ addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ HoodieWriteConfig cfg = cfgBuilder.build();
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+
+ String instantTime0 = "000";
+ client.startCommitWithTime(instantTime0);
+ List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
+ assertTrue(client.commit(instantTime0, client.bulkInsert(transformInputFn.apply(records0), instantTime0)), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime0), "After explicit commit, commit file should be created");
+
+ // Read from commit file
+ HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
+ createMetaClient().reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
+ HoodieCommitMetadata.class);
+ int inserts = 0;
+ for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+ for (HoodieWriteStat stat : pstat.getValue()) {
+ inserts += stat.getNumInserts();
+ }
+ }
+ assertEquals(200, inserts);
+
+ // Update + Inserts such that they just expand file1
+ String instantTime1 = "001";
+ client.startCommitWithTime(instantTime1);
+ List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
+ assertTrue(client.commit(instantTime1, client.upsert(transformInputFn.apply(records1), instantTime1)), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime1), "After explicit commit, commit file should be created");
+
+ metaClient = createMetaClient();
+ // Read from commit file
+ metadata = HoodieCommitMetadata.fromBytes(
+ metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
+ HoodieCommitMetadata.class);
+ inserts = 0;
+ int upserts = 0;
+ for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+ for (HoodieWriteStat stat : pstat.getValue()) {
+ inserts += stat.getNumInserts();
+ upserts += stat.getNumUpdateWrites();
}
- assertFalse(partitionToKeys.get(partitionPath).contains(key), "key " + key + " is duplicate within partition " + partitionPath);
- partitionToKeys.get(partitionPath).add(key);
}
+ assertEquals(0, inserts);
+ assertEquals(200, upserts);
+ }
+
+ protected void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail, Function createBrokenClusteringClientFn) throws IOException {
+ try {
+ Properties properties = new Properties();
+ properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
+ properties.setProperty("hoodie.auto.commit", "false");
+ properties.setProperty("hoodie.clustering.inline.max.commits", "1");
+ properties.setProperty("hoodie.clustering.inline", "true");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
+ testInsertTwoBatches(true, "2015/03/16", properties, true, createBrokenClusteringClientFn);
+ assertFalse(shouldFail);
+ } catch (HoodieException e) {
+ assertEquals(CLUSTERING_FAILURE, e.getMessage());
+ assertTrue(shouldFail);
+ }
+ }
+
+ protected void testDeletesWithoutInserts(boolean populateMetaFields, Function transformInputFn, Function transformOutputFn) {
+ final String testPartitionPath = "2016/09/26";
+ final int insertSplitLimit = 100;
+ // setup the small file handling params
+ HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
+ TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
+ ? new Properties() : getPropertiesForKeyGen());
+ dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
+ BaseHoodieWriteClient client = getHoodieWriteClient(config);
+ // delete non existent keys
+ String commitTime1 = "001";
+ client.startCommitWithTime(commitTime1);
+
+ List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
+ List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
+ transformOutputFn.apply(client.delete(transformInputFn.apply(hoodieKeysToDelete), commitTime1));
+ }
+
+ protected void testInlineScheduleClustering(Function createBrokenClusteringClientFn, HoodieClusteringConfig clusteringConfig,
+ Function transformInputFn, Function transformOutputFn) throws IOException {
+ testInsertTwoBatches(true, createBrokenClusteringClientFn);
+ HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
+ .withClusteringConfig(clusteringConfig)
+ .withProps(getPropertiesForKeyGen()).build();
+ dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
+ generateInsertsAndCommit(config, transformInputFn, transformOutputFn);
+ HoodieTableMetaClient metaClient = createMetaClient();
+ getAndAssertPendingClusteringPlans(clusteringConfig.getBoolean(SCHEDULE_INLINE_CLUSTERING), metaClient);
+ }
+
+ protected void testAndValidateClusteringOutputFiles(Function createBrokenClusteringClientFn, HoodieClusteringConfig clusteringConfig,
+ Function transformInputFn, Function transformOutputFn) throws IOException {
+ String partitionPath = "2015/03/16";
+ testInsertTwoBatches(true, partitionPath, createBrokenClusteringClientFn);
+ // Trigger clustering
+ HoodieWriteConfig config = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
+ .withClusteringConfig(clusteringConfig).build();
+ generateInsertsAndCommit(config, transformInputFn, transformOutputFn);
+ verifyClusteredFilesWithReplaceCommitMetadata(partitionPath);
+ }
+
+ private void generateInsertsAndCommit(HoodieWriteConfig config, Function transformInputFn, Function transformOutputFn) {
+ try (BaseHoodieWriteClient client = getHoodieWriteClient(config)) {
+ String commitTime = client.createNewInstantTime();
+ List<HoodieRecord> records = dataGen.generateInserts(commitTime, 200);
+ client.startCommitWithTime(commitTime);
+ List<WriteStatus> statusList = (List<WriteStatus>) transformOutputFn.apply(client.insert(transformInputFn.apply(records), commitTime));
+ assertNoWriteErrors(statusList);
+ client.commit(commitTime, transformInputFn.apply(statusList));
+ }
+ }
+
+ /**
+ * Test Deduplication Logic for write function.
+ *
+ * @param writeFn One of HoodieWriteClient non-prepped write APIs
+ * @throws Exception in case of failure
+ */
+ protected void testDeduplication(Function3<List<WriteStatus>, BaseHoodieWriteClient, List<HoodieRecord>, String> writeFn,
+ boolean populateMetaFields, boolean allowOperationMetadataField) throws Exception {
+ String newCommitTime = "001";
+ List<HoodieRecord> recordList = generate3AvroRecords(newCommitTime);
+ HoodieData<HoodieRecord> records = HoodieListData.eager(recordList);
+ HoodieWriteConfig writeConfig = getWriteConfigWithPopulateMetaFieldsAndAllowOperationMetaField(populateMetaFields, allowOperationMetadataField);
+ // Global dedup should be done based on recordKey only
+ List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupForCopyOnWriteStorage(writeConfig, records, true, allowOperationMetadataField ? 100 : 2, records.getNumPartitions());
+ if (allowOperationMetadataField) {
+ assertEquals(dedupedRecs.get(0).getOperation(), recordList.get(2).getOperation());
+ } else {
+ assertEquals(dedupedRecs.get(0).getPartitionPath(), recordList.get(2).getPartitionPath());
+ // non-Global dedup should be done based on both recordKey and partitionPath
+ dedupForCopyOnWriteStorage(writeConfig, records, false, 2, records.getNumPartitions());
+ }
+ // Perform write-action and check
+ try (BaseHoodieWriteClient client = getHoodieWriteClient(writeConfig)) {
+ client.startCommitWithTime(newCommitTime);
+ List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
+ assertNoWriteErrors(statuses);
+ assertEquals(2, statuses.size());
+ assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
+ .collect(Collectors.toList()));
+ }
+ }
+
+ /**
+ * Test auto-commit by applying the write function.
+ *
+ * @param writeFn One of HoodieWriteClient Write API
+ * @throws Exception in case of failure
+ */
+ protected void testAutoCommit(Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+ boolean isPrepped, boolean populateMetaFields) throws Exception {
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false); // Set autoCommit false
+ addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ try (BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
+ String prevCommitTime = "000";
+ String newCommitTime = "001";
+ int numRecords = 200;
+ Object result = castInsertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn, isPrepped, false, numRecords);
+ assertFalse(testTable.commitExists(newCommitTime), "If Autocommit is false, then commit should not be made automatically");
+ assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
+ assertTrue(testTable.commitExists(newCommitTime), "After explicit commit, commit file should be created");
+ }
+ }
+
+ /**
+ * When records getting inserted are deleted in the same write batch, hudi should have deleted those records and
+ * not be available in read path.
+ *
+ * @throws Exception
+ */
+ protected void testDeletesForInsertsInSameBatch() throws Exception {
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build());
+ /**
+ * Write 200 inserts and issue deletes to a subset(50) of inserts.
+ */
+ String initCommitTime = "000";
+ String newCommitTime = "001";
+
+ final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ (String instantTime, Integer numRecordsInThisCommit) -> {
+ List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
+ List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
+
+ recordsInFirstBatch.addAll(fewRecordsForInsert);
+ recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+ return recordsInFirstBatch;
+ };
+
+ castWriteBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime, -1, recordGenFunction,
+ BaseHoodieWriteClient::upsert, true, 150, 150, 1, false, true);
+ }
+
+ /**
+ * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
+ *
+ * @throws Exception in case of error
+ */
+ protected void testHoodieConcatHandle(boolean populateMetaFields, boolean isPrepped) throws Exception {
+ metaClient = createMetaClient(basePath);
+ // Force using older timeline layout
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
+ addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ HoodieWriteConfig hoodieWriteConfig = cfgBuilder.withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(VERSION_0).build();
+ HoodieTableMetaClient.withPropertyBuilder().fromMetaClient(metaClient).setTimelineLayoutVersion(VERSION_0)
+ .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
+
+ BaseHoodieWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
+
+ // Write 1 (only inserts)
+ String newCommitTime = "001";
+ String initCommitTime = "000";
+ int numRecords = 200;
+ castInsertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, BaseHoodieWriteClient::insert,
+ isPrepped, true, numRecords, populateMetaFields);
+
+ // Write 2 (updates)
+ String prevCommitTime = newCommitTime;
+ newCommitTime = "004";
+ numRecords = 100;
+ String commitTimeBetweenPrevAndNew = "002";
+
+ final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
+
+ castWriteBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, recordGenFunction,
+ BaseHoodieWriteClient::insert, true, numRecords, 300, 2, false, populateMetaFields);
+ }
+
+ protected void testHoodieConcatHandleOnDupInserts(boolean isPrepped) throws Exception {
+ HoodieWriteConfig config = getConfigBuilder()
+ .withMergeAllowDuplicateOnInserts(true)
+ .build();
+
+ BaseHoodieWriteClient client = getHoodieWriteClient(config);
+
+ // Write 1 (only inserts)
+ String initCommitTime = "000";
+ String newCommitTime = "001";
+ int firstInsertRecords = 50;
+ castInsertFirstBatch(config, client, newCommitTime, initCommitTime, firstInsertRecords, BaseHoodieWriteClient::insert,
+ isPrepped, true, firstInsertRecords, config.populateMetaFields());
+
+ // Write 2 (updates with duplicates)
+ String prevCommitTime = newCommitTime;
+ newCommitTime = "004";
+ int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
+ List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
+
+ final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ generateWrapRecordsFn(isPrepped, config, dataGen::generateUpdates);
+
+ castWriteBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
+ secondInsertRecords, recordGenFunction, BaseHoodieWriteClient::insert, true, secondInsertRecords,
+ firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+ }
+
+ /**
+ * Test one of HoodieWriteClient upsert(Prepped) APIs.
+ *
+ * @param writeFn One of Hoodie Write Function API
+ * @throws Exception in case of error
+ */
+ protected void testUpsertsInternal(Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean populateMetaFields, boolean isPrepped) throws Exception {
+ metaClient = createMetaClient();
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withRollbackUsingMarkers(true);
+ addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ // Force using older timeline layout
+ HoodieWriteConfig config = cfgBuilder.withTimelineLayoutVersion(VERSION_0).build();
+ HoodieTableMetaClient.withPropertyBuilder().fromMetaClient(metaClient).setTimelineLayoutVersion(VERSION_0)
+ .setPopulateMetaFields(populateMetaFields).initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
+
+ BaseHoodieWriteClient client = getHoodieWriteClient(config);
+
+ // Write 1 (only inserts)
+ String newCommitTime = "001";
+ String initCommitTime = "000";
+ int numRecords = 200;
+ castInsertFirstBatch(config, client, newCommitTime, initCommitTime, numRecords, BaseHoodieWriteClient::insert,
+ isPrepped, true, numRecords, populateMetaFields);
+
+ // Write 2 (updates)
+ String prevCommitTime = newCommitTime;
+ newCommitTime = "004";
+ numRecords = 100;
+ String commitTimeBetweenPrevAndNew = "002";
+ castUpdateBatch(config, client, newCommitTime, prevCommitTime,
+ Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
+ numRecords, 200, 2, populateMetaFields);
+
+ // Delete 1
+ prevCommitTime = newCommitTime;
+ newCommitTime = "005";
+ numRecords = 50;
+
+ castDeleteBatch(config, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
+ 0, 150, config.populateMetaFields());
+
+ // Now simulate an upgrade and perform a restore operation
+ HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
+ TimelineLayoutVersion.CURR_VERSION).build();
+ client = getHoodieWriteClient(newConfig);
+
+ client.savepoint("004", "user1", "comment1");
+
+ client.restoreToInstant("004", config.isMetadataTableEnabled());
+
+ assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
+
+ client.deleteSavepoint("004");
+ assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
+
+ assertTheEntireDatasetHasAllRecordsStill(200);
+
+ // Perform Delete again on upgraded dataset.
+ prevCommitTime = newCommitTime;
+ newCommitTime = "006";
+ numRecords = 50;
+
+ castDeleteBatch(newConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true, 0, 150);
+
+ checkTimelineForUpsertsInternal(metaClient);
+
+ testMergeHandle(config);
+ }
+
+ /**
+ * Tests deletion of records.
+ */
+ protected void testDeletes(Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn,
+ int numRecordsInSecondBatch, int expRecordsInSecondBatch, int expTotalRecords) throws Exception {
+ HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
+ BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build());
+ /**
+ * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
+ */
+ String initCommitTime = "000";
+ String newCommitTime = "001";
+
+ final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ (String instantTime, Integer numRecordsInThisCommit) -> {
+ List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
+ List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
+
+ recordsInFirstBatch.addAll(fewRecordsForInsert);
+ recordsInFirstBatch.addAll(fewRecordsForDelete);
+ return recordsInFirstBatch;
+ };
+
+ castWriteBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
+ // unused as genFn uses hard-coded number of inserts/updates/deletes
+ -1, recordGenFunction, BaseHoodieWriteClient::upsert, true, 200, 200, 1, false, true);
+
+ /**
+ * Write 2 (deletes+writes).
+ */
+ String prevCommitTime = newCommitTime;
+ newCommitTime = "004";
+ recordGenFunction = secondBatchGenFn.apply(newCommitTime, numRecordsInSecondBatch, recordsInFirstBatch);
+
+ castWriteBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, numRecordsInSecondBatch, recordGenFunction,
+ BaseHoodieWriteClient::upsert, true, expRecordsInSecondBatch, expTotalRecords, 2, false, true);
+ }
+
+ protected void testRollbackFailedCommits(boolean populateMetaFields) throws Exception {
+ // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
+ HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
+ HoodieTestUtils.init(storageConf, basePath);
+ BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+
+ // perform 1 successful commit
+ castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, true);
+
+ // Perform 2 failed writes to table
+ castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ // refresh data generator to delete records generated from failed commits
+ dataGen = new HoodieTestDataGenerator();
+ // Perform 1 successful write
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, true);
+ HoodieTableMetaClient metaClient = createMetaClient();
+
+ assertEquals(0, metaClient.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ assertEquals(2, metaClient.getActiveTimeline().filterInflights().countInstants());
+ assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants());
+ // Wait till enough time passes such that the first 2 failed commits heartbeats are expired
+ boolean conditionMet = false;
+ while (!conditionMet) {
+ conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
+ Thread.sleep(2000);
+ }
+ client.close();
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ // Perform 1 successful write
+ castWriteBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, true);
+ client.clean();
+ client.close();
+ HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+ if (cleaningPolicy.isLazy()) {
+ assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ // Since we write rollbacks not clean, there should be no clean action on the timeline
+ assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+ assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+ } else if (cleaningPolicy.isNever()) {
+ // never will get translated to Lazy if OCC is enabled.
+ assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ // There should be no clean or rollback action on the timeline
+ assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+ assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+ }
+ }
+
+ protected void testRollbackFailedCommitsToggleCleaningPolicy(boolean populateMetaFields) throws Exception {
+ HoodieTestUtils.init(storageConf, basePath);
+ metaClient = createMetaClient();
+ HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
+ BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ // Perform 1 successful writes to table
+ castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, true);
+
+ // Perform 1 failed writes to table
+ castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ // Toggle cleaning policy to LAZY
+ cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
+ // Perform 2 failed writes to table
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ // Wait till enough time passes such that the 2 failed commits heartbeats are expired
+ boolean conditionMet = false;
+ while (!conditionMet) {
+ conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
+ Thread.sleep(2000);
+ }
+ client.clean();
+ HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+ assertEquals(3, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ // Perform 2 failed commits
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+ 0, false);
+ client.close();
+ // Toggle cleaning policy to EAGER
+ cleaningPolicy = EAGER;
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ client.startCommit();
+ client.close();
+ timeline = metaClient.getActiveTimeline().reload();
+ // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
+ assertEquals(3, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ assertEquals(1, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+ }
+
+ protected void testParallelInsertAndCleanPreviousFailedCommits(boolean populateMetaFields) throws Exception {
+ HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
+ ExecutorService service = Executors.newFixedThreadPool(2);
+ HoodieTestUtils.init(storageConf, basePath);
+ BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ // perform 1 successful write
+ castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+ 0, true);
+
+ // Perform 2 failed writes to table
+ castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+ 0, false);
+ client.close();
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+ 100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+ 0, false);
+ client.close();
+ // refresh data generator to delete records generated from failed commits
+ dataGen = new HoodieTestDataGenerator();
+ // Create a successful commit
+ Future<Object> commit3 = service.submit(() -> castWriteBatch(getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)),
+ "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
+ BaseHoodieWriteClient::bulkInsert, false, 100, 100, 0, true));
+ commit3.get();
+ HoodieTableMetaClient metaClient = createMetaClient();
+
+ assertEquals(0, metaClient.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ assertEquals(2, metaClient.getActiveTimeline().filterInflights().countInstants());
+ assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants());
+ client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+ // Wait till enough time passes such that the first 2 failed commits heartbeats are expired
+ boolean conditionMet = false;
+ while (!conditionMet) {
+ conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
+ Thread.sleep(2000);
+ }
+ Future<Object> commit4 = service.submit(() -> castWriteBatch(getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)),
+ "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
+ BaseHoodieWriteClient::bulkInsert, false, 100, 100, 0, true));
+ Future<HoodieCleanMetadata> clean1 = service.submit(() -> getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)).clean());
+ commit4.get();
+ clean1.get();
+ client.close();
+ HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+ assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+ // Since we write rollbacks not clean, there should be no clean action on the timeline
+ assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+ assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+ service.shutdown();
}
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index 1e6e74664d8..39db790d795 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -1795,7 +1795,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
context = new HoodieJavaEngineContext(storageConf);
- HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+ HoodieWriteConfig config = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
HoodieJavaWriteClient client = getHoodieWriteClient(config);
// Write 1 (Bulk insert)
@@ -1869,7 +1869,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
context = new HoodieJavaEngineContext(storageConf);
- HoodieWriteConfig initialConfig = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+ HoodieWriteConfig initialConfig = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
.withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build()).build();
HoodieJavaWriteClient client = getHoodieWriteClient(config);
@@ -2511,23 +2511,6 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
}
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
- HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
- return builder.withCompactionConfig(
- HoodieCompactionConfig.newBuilder()
- .compactionSmallFileSize(smallFileSize)
- // Set rollback to LAZY so no inflights are deleted
- .insertSplitSize(insertSplitSize).build())
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
- .withStorageConfig(
- HoodieStorageConfig.newBuilder()
- .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
- .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
- .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
- .build();
- }
-
public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, HoodieIndex.IndexType indexType,
HoodieFailedWritesCleaningPolicy cleaningPolicy) {
return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schemaStr)
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index a6204ca75e6..3baf26257b8 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -18,88 +18,33 @@
package org.apache.hudi.client.functional;
-import org.apache.hudi.avro.model.HoodieCleanMetadata;
-import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.client.HoodieJavaWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy;
import org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy;
import org.apache.hudi.client.common.JavaTaskContextSupplier;
-import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
-import org.apache.hudi.common.config.HoodieStorageConfig;
-import org.apache.hudi.common.config.LockConfiguration;
-import org.apache.hudi.common.config.TypedProperties;
-import org.apache.hudi.common.data.HoodieData;
-import org.apache.hudi.common.data.HoodieListData;
import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.fs.FSUtils;
-import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
-import org.apache.hudi.common.model.HoodieFileGroupId;
-import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodieOperation;
-import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordDelegate;
import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
-import org.apache.hudi.common.model.HoodieWriteStat;
-import org.apache.hudi.common.model.IOType;
-import org.apache.hudi.common.model.WriteConcurrencyMode;
-import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.marker.MarkerType;
-import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
-import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestTable;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.testutils.RawTripTestPayload;
-import org.apache.hudi.common.util.ClusteringUtils;
-import org.apache.hudi.common.util.CollectionUtils;
-import org.apache.hudi.common.util.MarkerUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieClusteringConfig;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieLockConfig;
-import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieCorruptedDataException;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieUpsertException;
-import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.index.HoodieIndex.IndexType;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.keygen.BaseKeyGenerator;
-import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.HoodieWriteMetadata;
-import org.apache.hudi.table.action.commit.HoodieWriteHelper;
-import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -109,47 +54,18 @@ import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
+import java.util.function.Function;
import java.util.stream.Stream;
-import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
-import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
-import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
-import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
-import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
-import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
@SuppressWarnings("unchecked")
public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTestHarness {
- private static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
-
private static Stream<Arguments> rollbackAfterConsistencyCheckFailureParams() {
return Stream.of(
Arguments.of(true),
@@ -157,19 +73,79 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
);
}
- private HoodieTestTable testTable;
-
@BeforeEach
public void setUpTestTable() {
testTable = HoodieMetadataTestTable.of(metaClient);
}
+ private static final Function<Object, Object> IDENTITY = Function.identity();
+
+ private final Function<HoodieWriteConfig, BaseHoodieWriteClient> createBrokenClusteringClient =
+ config -> new WriteClientBrokenClustering<>(context, config);
+
+ private final Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTable =
+ (metaClient, config) -> getHoodieTable(metaClient, config);
+
+ @Override
+ protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+ return insertFirstBatch(writeConfig, (HoodieJavaWriteClient) client, newCommitTime, initCommitTime, numRecordsInThisCommit,
+ (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+ isPreppedAPI, assertForCommit, expRecordsInThisCommit, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ return writeBatch((HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+ (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+ assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates);
+
+ return writeBatch((HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+ (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime), assertForCommit, expRecordsInThisCommit, expTotalRecords,
+ expTotalCommits, false, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+ return deleteBatch(writeConfig, (HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit,
+ isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+ String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
+ for (int i = 0; i < fullPartitionPaths.length; i++) {
+ fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
+ }
+ assertEquals(200, countRowsInPaths(basePath, storage, fullPartitionPaths), "Must contain " + 200 + " records");
+ return fullPartitionPaths;
+ }
+
/**
* Test Auto Commit behavior for HoodieWriteClient insert API.
*/
@Test
public void testAutoCommitOnInsert() throws Exception {
- testAutoCommit(HoodieJavaWriteClient::insert, false, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insert(recordRDD, instantTime), false, true);
}
/**
@@ -177,7 +153,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testAutoCommitOnInsertPrepped() throws Exception {
- testAutoCommit(HoodieJavaWriteClient::insertPreppedRecords, true, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insertPreppedRecords(recordRDD, instantTime), true, true);
}
/**
@@ -185,7 +161,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testAutoCommitOnUpsert() throws Exception {
- testAutoCommit(HoodieJavaWriteClient::upsert, false, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), false, true);
}
/**
@@ -193,7 +169,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testAutoCommitOnUpsertPrepped() throws Exception {
- testAutoCommit(HoodieJavaWriteClient::upsertPreppedRecords, true, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
}
/**
@@ -201,7 +177,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testAutoCommitOnBulkInsert() throws Exception {
- testAutoCommit(HoodieJavaWriteClient::bulkInsert, false, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsert(recordRDD, instantTime), false, true);
}
/**
@@ -213,48 +189,12 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
Option.empty()), true, true);
}
- /**
- * Test auto-commit by applying the write function.
- *
- * @param writeFn One of HoodieWriteClient Write API
- * @throws Exception in case of failure
- */
- private void testAutoCommit(Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn,
- boolean isPrepped, boolean populateMetaFields) throws Exception {
- // Set autoCommit false
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
- try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());) {
-
- String prevCommitTime = "000";
- String newCommitTime = "001";
- int numRecords = 200;
- List<WriteStatus> result = insertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn,
- isPrepped, false, numRecords);
-
- assertFalse(testTable.commitExists(newCommitTime),
- "If Autocommit is false, then commit should not be made automatically");
- assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
- assertTrue(testTable.commitExists(newCommitTime),
- "After explicit commit, commit file should be created");
- }
- }
-
- private void insertWithConfig(HoodieWriteConfig config, int numRecords, String instant) throws Exception {
- try (HoodieJavaWriteClient client = getHoodieWriteClient(config)) {
- Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
- writeClient.bulkInsert(recordRDD, instantTime, Option.empty());
- List<WriteStatus> result = insertFirstBatch(config, client, instant,
- "000", numRecords, writeFn, false, false, numRecords);
- }
- }
-
/**
* Test De-duplication behavior for HoodieWriteClient insert API.
*/
@Test
public void testDeduplicationOnInsert() throws Exception {
- testDeduplication(HoodieJavaWriteClient::insert, true);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.insert(records, commitTime), true, false);
}
/**
@@ -262,7 +202,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testDeduplicationKeepOperationFieldOnInsert() throws Exception {
- testDeduplicationKeepOperation(HoodieJavaWriteClient::insert, true);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.insert(records, commitTime), true, true);
}
/**
@@ -270,7 +210,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testDeduplicationOnBulkInsert() throws Exception {
- testDeduplication(HoodieJavaWriteClient::bulkInsert, true);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.bulkInsert(records, commitTime), true, false);
}
/**
@@ -278,143 +218,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testDeduplicationOnUpsert() throws Exception {
- testDeduplication(HoodieJavaWriteClient::upsert, true);
- }
-
- /**
- * Test Deduplication Logic for write function.
- *
- * @param writeFn One of HoodieWriteClient non-prepped write APIs
- * @throws Exception in case of failure
- */
- private void testDeduplication(
- Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
- String newCommitTime = "001";
-
- String recordKey = UUID.randomUUID().toString();
- HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
- HoodieRecord<RawTripTestPayload> recordOne =
- new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime));
-
- HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
- HoodieRecord recordTwo =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
- // Same key and partition as keyTwo
- HoodieRecord recordThree =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
- HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieListData.eager(Arrays.asList(recordOne, recordTwo, recordThree));
- HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
- .combineInput(true, true);
- addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
- HoodieWriteConfig writeConfig = configBuilder.build();
-
- // Global dedup should be done based on recordKey only
- HoodieIndex index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(true);
- int dedupParallelism = records.getNumPartitions() + 100;
- HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(records.getNumPartitions(), dedupedRecsRdd.getNumPartitions());
- assertEquals(1, dedupedRecs.size());
- assertEquals(dedupedRecs.get(0).getPartitionPath(), recordThree.getPartitionPath());
- assertNodupesWithinPartition(dedupedRecs);
-
- // non-Global dedup should be done based on both recordKey and partitionPath
- index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(false);
- dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(records.getNumPartitions(), dedupedRecsRdd.getNumPartitions());
- assertEquals(2, dedupedRecs.size());
- assertNodupesWithinPartition(dedupedRecs);
-
- // Perform write-action and check
- List<HoodieRecord> recordList = Arrays.asList(recordOne, recordTwo, recordThree);
-
- try (HoodieJavaWriteClient client = getHoodieWriteClient(writeConfig)) {
- client.startCommitWithTime(newCommitTime);
- List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
- assertNoWriteErrors(statuses);
- assertEquals(2, statuses.size());
- assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
- .collect(Collectors.toList()));
- }
- }
-
- /**
- * Test Deduplication Logic for write function.
- *
- * @param writeFn One of HoodieWriteClient non-prepped write APIs
- * @throws Exception in case of failure
- */
- private void testDeduplicationKeepOperation(
- Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
- String newCommitTime = "001";
-
- String recordKey = UUID.randomUUID().toString();
- HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
- HoodieRecord<RawTripTestPayload> recordOne =
- new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime), HoodieOperation.INSERT);
-
- HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
- HoodieRecord recordTwo =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.INSERT);
-
- // Same key and partition as keyTwo
- HoodieRecord recordThree =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.UPDATE_AFTER);
-
- HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieListData.eager(Arrays.asList(recordOne, recordTwo, recordThree));
- HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAllowOperationMetadataField(true)
- .combineInput(true, true);
- addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
- HoodieWriteConfig writeConfig = configBuilder.build();
-
- // Global dedup should be done based on recordKey only
- HoodieIndex index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(true);
- int dedupParallelism = records.getNumPartitions() + 100;
- HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(dedupedRecs.get(0).getOperation(), recordThree.getOperation());
-
- // Perform write-action and check
- List<HoodieRecord> recordList = Arrays.asList(recordOne, recordTwo, recordThree);
-
- try (HoodieJavaWriteClient client = getHoodieWriteClient(writeConfig)) {
- client.startCommitWithTime(newCommitTime);
- List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
- assertNoWriteErrors(statuses);
- assertEquals(2, statuses.size());
- assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
- .collect(Collectors.toList()));
- }
- }
-
- /**
- * Assert that there is no duplicate key at the partition level.
- *
- * @param recordDelegates List of Hoodie record delegates
- */
- void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
- Map<String, Set<String>> partitionToKeys = new HashMap<>();
- for (HoodieRecordDelegate r : recordDelegates) {
- String recordKey = r.getRecordKey();
- String partitionPath = r.getPartitionPath();
- if (!partitionToKeys.containsKey(partitionPath)) {
- partitionToKeys.put(partitionPath, new HashSet<>());
- }
- assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
- partitionToKeys.get(partitionPath).add(recordKey);
- }
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.upsert(records, commitTime), true, false);
}
/**
@@ -422,9 +226,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testUpserts() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsert, false);
+ testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), true, false);
}
/**
@@ -432,134 +234,23 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testUpsertsPrepped() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsertPreppedRecords, true);
+ testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
}
- /**
- * Test one of HoodieWriteClient upsert(Prepped) APIs.
- *
- * @param config Write Config
- * @param writeFn One of Hoodie Write Function API
- * @throws Exception in case of error
- */
- private void testUpsertsInternal(HoodieWriteConfig config,
- Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPrepped)
- throws Exception {
- // Force using older timeline layout
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
- .withRollbackUsingMarkers(true)
- .withProps(config.getProps()).withTimelineLayoutVersion(
- VERSION_0).build();
-
- HoodieTableMetaClient.withPropertyBuilder()
- .fromMetaClient(metaClient)
- .setTimelineLayoutVersion(VERSION_0)
- .setPopulateMetaFields(config.populateMetaFields())
- .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
- HoodieJavaWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String newCommitTime = "001";
- String initCommitTime = "000";
- int numRecords = 200;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, HoodieJavaWriteClient::insert,
- isPrepped, true, numRecords, config.populateMetaFields());
-
- // Write 2 (updates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- numRecords = 100;
- String commitTimeBetweenPrevAndNew = "002";
- updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime,
- Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
- numRecords, 200, 2, config.populateMetaFields());
-
- // Delete 1
- prevCommitTime = newCommitTime;
- newCommitTime = "005";
- numRecords = 50;
-
- deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
- 0, 150, config.populateMetaFields());
-
- // Now simulate an upgrade and perform a restore operation
- HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
- TimelineLayoutVersion.CURR_VERSION).build();
- client = getHoodieWriteClient(newConfig);
-
- client.savepoint("004", "user1", "comment1");
-
- client.restoreToInstant("004", config.isMetadataTableEnabled());
-
- assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
-
- client.deleteSavepoint("004");
- assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
-
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(200, countRowsInPaths(basePath, storage, fullPartitionPaths),
- "Must contain " + 200 + " records");
-
- // Perform Delete again on upgraded dataset.
- prevCommitTime = newCommitTime;
- newCommitTime = "006";
- numRecords = 50;
-
- deleteBatch(newConfig, client, newCommitTime, prevCommitTime,
- initCommitTime, numRecords, isPrepped, true,
- 0, 150);
-
- HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
- List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
- assertEquals(5, instants.size());
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"),
- instants.get(0));
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"),
- instants.get(1));
- // New Format should have all states of instants
- assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"),
- instants.get(2));
- assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"),
- instants.get(3));
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"),
- instants.get(4));
-
- final HoodieWriteConfig cfg = hoodieWriteConfig;
+ @Override
+ protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
final String instantTime = "007";
HoodieTableMetaClient metaClient = createMetaClient();
- String basePathStr = basePath;
- HoodieTable table = getHoodieTable(metaClient, cfg);
- String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
- metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
- HoodieCommitMetadata.class);
- String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
- .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
- .map(ee -> ee.getPath()).orElse(null);
- String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
- .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
- .map(ee -> ee.getPartitionPath()).orElse(null);
- Path baseFilePath = new Path(basePathStr, filePath);
- HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath.toString());
+ HoodieTable table = getHoodieTable(metaClient, config);
+ Pair<String, String> partitionAndBaseFilePaths = getPartitionAndBaseFilePathsFromLatestCommitMetadata(metaClient);
+ HoodieBaseFile baseFile = new HoodieBaseFile(partitionAndBaseFilePaths.getRight());
HoodieMergeHandle handle = null;
try {
- handle = new HoodieMergeHandle(cfg, instantTime, table, new HashMap<>(),
- partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new JavaTaskContextSupplier(),
+ handle = new HoodieMergeHandle(config, instantTime, table, new HashMap<>(),
+ partitionAndBaseFilePaths.getLeft(), FSUtils.getFileId(baseFile.getFileName()), baseFile, new JavaTaskContextSupplier(),
config.populateMetaFields() ? Option.empty() :
- Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
- WriteStatus writeStatus = new WriteStatus(false, 0.0);
- writeStatus.setStat(new HoodieWriteStat());
- writeStatus.getStat().setNumWrites(0);
- handle.performMergeDataValidationCheck(writeStatus);
+ Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(config.getProps())));
} catch (HoodieCorruptedDataException e1) {
fail("Exception not expected because merge validation check is disabled");
} finally {
@@ -571,13 +262,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
handle = null;
try {
final String newInstantTime = "006";
- cfg.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
- HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(cfg.getProps()).build();
+ config.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
+ HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(config.getProps()).build();
// does the handle need to be closed to clean up the writer it contains?
handle = new HoodieMergeHandle(cfg2, newInstantTime, table, new HashMap<>(),
- partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new JavaTaskContextSupplier(),
+ partitionAndBaseFilePaths.getLeft(), FSUtils.getFileId(baseFile.getFileName()), baseFile, new JavaTaskContextSupplier(),
config.populateMetaFields() ? Option.empty() :
- Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+ Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(config.getProps())));
fail("The above line should have thrown an exception");
} catch (HoodieUpsertException e2) {
// expected
@@ -593,58 +284,15 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testInsertsWithHoodieConcatHandle() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testHoodieConcatHandle(cfgBuilder.build(), false);
+ testHoodieConcatHandle(true, false);
}
/**
* Test InsertPrepped API for HoodieConcatHandle.
*/
+ @Test
public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testHoodieConcatHandle(cfgBuilder.build(), true);
- }
-
- /**
- * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
- *
- * @param config Write Config
- * @throws Exception in case of error
- */
- private void testHoodieConcatHandle(HoodieWriteConfig config, boolean isPrepped)
- throws Exception {
- // Force using older timeline layout
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
- .withProps(config.getProps()).withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(
- VERSION_0).build();
- HoodieTableMetaClient.withPropertyBuilder()
- .fromMetaClient(metaClient)
- .setTimelineLayoutVersion(VERSION_0)
- .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
- HoodieJavaWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String newCommitTime = "001";
- String initCommitTime = "000";
- int numRecords = 200;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, HoodieJavaWriteClient::insert,
- isPrepped, true, numRecords, config.populateMetaFields());
-
- // Write 2 (updates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- numRecords = 100;
- String commitTimeBetweenPrevAndNew = "002";
-
- final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
-
- writeBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime,
- numRecords, recordGenFunction, HoodieJavaWriteClient::insert, true, numRecords, 300,
- 2, false, config.populateMetaFields());
+ testHoodieConcatHandle(true, true);
}
/**
@@ -652,8 +300,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testInsertsWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), false);
+ testHoodieConcatHandleOnDupInserts(false);
}
/**
@@ -661,83 +308,25 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testInsertsPreppedWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), true);
- }
-
- private void testHoodieConcatHandleOnDupInserts(HoodieWriteConfig config, boolean isPrepped) throws Exception {
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
- .withProps(config.getProps())
- .withMergeAllowDuplicateOnInserts(true)
- .build();
-
- HoodieJavaWriteClient<RawTripTestPayload> client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String initCommitTime = "000";
- String newCommitTime = "001";
- int firstInsertRecords = 50;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, firstInsertRecords, HoodieJavaWriteClient::insert,
- isPrepped, true, firstInsertRecords, config.populateMetaFields());
-
- // Write 2 (updates with duplicates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
- List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
-
- final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUpdates);
-
- writeBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
- secondInsertRecords, recordGenFunction, HoodieJavaWriteClient::insert, true, secondInsertRecords,
- firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+ testHoodieConcatHandleOnDupInserts(true);
}
/**
* Tests deletion of records.
*/
+ @Test
public void testDeletes() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());
- /**
- * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
- */
- String initCommitTime = "000";
- String newCommitTime = "001";
-
- final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
- Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
- List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
-
- recordsInFirstBatch.addAll(fewRecordsForInsert);
- recordsInFirstBatch.addAll(fewRecordsForDelete);
- return recordsInFirstBatch;
+ Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn =
+ (String instantTime, Integer numRecordsInThisCommit, List<HoodieRecord> recordsInFirstBatch) -> {
+ List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
+ return (time, numRecords) -> {
+ List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
+ recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+ recordsInSecondBatch.addAll(dataGen.generateInserts(instantTime, 25));
+ return recordsInSecondBatch;
+ };
};
- writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
- // unused as genFn uses hard-coded number of inserts/updates/deletes
- -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 200, 200, 1, false,
- true);
-
- /**
- * Write 2 (deletes+writes).
- */
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- final List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
-
- recordGenFunction = (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
- recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
- recordsInSecondBatch.addAll(dataGen.generateInserts(instantTime, 25));
- return recordsInSecondBatch;
- };
- writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 75, recordGenFunction,
- HoodieJavaWriteClient::upsert, true, 25, 175, 2, false,
- true);
+ super.testDeletes(secondBatchGenFn, 75, 25, 175);
}
/**
@@ -748,367 +337,41 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testDeletesForInsertsInSameBatch() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());
- /**
- * Write 200 inserts and issue deletes to a subset(50) of inserts.
- */
- String initCommitTime = "000";
- String newCommitTime = "001";
-
- final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
- Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
- List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
-
- recordsInFirstBatch.addAll(fewRecordsForInsert);
- recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
- return recordsInFirstBatch;
- };
-
- writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
- -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 150, 150, 1, false,
- true);
+ super.testDeletesForInsertsInSameBatch();
}
@Test
public void testAndValidateClusteringOutputFiles() throws IOException {
- String partitionPath = "2015/03/16";
- testInsertTwoBatches(true, partitionPath);
-
- // Trigger clustering
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
- .withClusteringConfig(HoodieClusteringConfig.newBuilder()
- .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
- .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
- .withInlineClustering(true).withInlineClusteringNumCommits(2).build());
- try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
- int numRecords = 200;
- String newCommitTime = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(newCommitTime, numRecords);
- client.startCommitWithTime(newCommitTime);
- List<WriteStatus> statuses = client.insert(records1, newCommitTime);
- client.commit(newCommitTime, statuses);
- assertNoWriteErrors(statuses);
-
- metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieInstant replaceCommitInstant =
- metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
- HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
- .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
- HoodieReplaceCommitMetadata.class);
-
- List<String> filesFromReplaceCommit = new ArrayList<>();
- replaceCommitMetadata.getPartitionToWriteStats()
- .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
-
- // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
- List<StoragePathInfo> pathInfoList =
- storage.listDirectEntries(new StoragePath(basePath + "/" + partitionPath));
- List<String> clusteredFiles = pathInfoList.stream()
- .filter(
- entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
- .map(fileStatus -> partitionPath + "/" + fileStatus.getPath().getName())
- .collect(Collectors.toList());
- assertEquals(clusteredFiles, filesFromReplaceCommit);
- }
+ HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder()
+ .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
+ .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
+ .withInlineClustering(true).withInlineClusteringNumCommits(2).build();
+ testAndValidateClusteringOutputFiles(createBrokenClusteringClient, clusteringConfig, IDENTITY, IDENTITY);
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testInlineScheduleClustering(boolean scheduleInlineClustering) throws IOException {
- testInsertTwoBatches(true);
-
- // setup clustering config.
HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
.withClusteringTargetPartitions(0).withAsyncClusteringMaxCommits(1).withInlineClustering(false).withScheduleInlineClustering(scheduleInlineClustering)
.withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
- .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
- .build();
-
- HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
- .withClusteringConfig(clusteringConfig)
- .withProps(getPropertiesForKeyGen()).build();
- HoodieJavaWriteClient client = getHoodieWriteClient(config);
- dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
- String commitTime1 = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
- client.startCommitWithTime(commitTime1);
- List<WriteStatus> statuses = client.insert(records1, commitTime1);
- assertNoWriteErrors(statuses);
- client.commit(commitTime1, statuses);
-
- HoodieTableMetaClient metaClient = createMetaClient();
- List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
- ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
- if (scheduleInlineClustering) {
- assertEquals(1, pendingClusteringPlans.size());
- } else {
- assertEquals(0, pendingClusteringPlans.size());
- }
- }
-
- private List<HoodieRecord> testInsertAndClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields,
- boolean completeClustering, boolean assertSameFileIds, String validatorClasses,
- String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation) throws Exception {
- Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields);
- testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords);
- return allRecords.getLeft().getLeft();
+ .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName()).build();
+ testInlineScheduleClustering(createBrokenClusteringClient, clusteringConfig, IDENTITY, IDENTITY);
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail) throws IOException {
- try {
- Properties properties = new Properties();
- properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
- properties.setProperty("hoodie.auto.commit", "false");
- properties.setProperty("hoodie.clustering.inline.max.commits", "1");
- properties.setProperty("hoodie.clustering.inline", "true");
- testInsertTwoBatches(true, "2015/03/16", properties, true);
- assertFalse(shouldFail);
- } catch (HoodieException e) {
- assertEquals(CLUSTERING_FAILURE, e.getMessage());
- assertTrue(shouldFail);
- }
- }
-
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields) throws IOException {
- return testInsertTwoBatches(populateMetaFields, "2015/03/16");
- }
-
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath) throws IOException {
- return testInsertTwoBatches(populateMetaFields, partitionPath, new Properties(), false);
- }
-
- /**
- * This method returns following three items:
- * 1. List of all HoodieRecord written in the two batches of insert.
- * 2. Commit instants of the two batches.
- * 3. List of new file group ids that were written in the two batches.
- */
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props,
- boolean failInlineClustering) throws IOException {
- // create config to not update small files.
- HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
- populateMetaFields ? props : getPropertiesForKeyGen());
- HoodieJavaWriteClient client;
- if (failInlineClustering) {
- if (null != writeClient) {
- writeClient.close();
- writeClient = null;
- }
- client = new WriteClientBrokenClustering(context, config);
- } else {
- client = getHoodieWriteClient(config);
- }
-
- dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
- String commitTime1 = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
- List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
- Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
-
- String commitTime2 = client.createNewInstantTime();
- List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
- List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
- client.close();
- Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
- Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
- fileIdsUnion.addAll(fileIds2);
- //verify new files are created for 2nd write
- Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
- fileIdIntersection.retainAll(fileIds2);
- assertEquals(0, fileIdIntersection.size());
- return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
- }
-
- private void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
- String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
- Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords) throws IOException {
-
- HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
- .withClusteringConfig(clusteringConfig)
- .withProps(getPropertiesForKeyGen()).build();
- HoodieWriteMetadata<List<WriteStatus>> clusterMetadata =
- performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords.getLeft());
- if (assertSameFileIds) {
- Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
- .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
- Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
- assertEquals(insertedFileIds, replacedFileIds);
- }
- if (completeClustering) {
- String clusteringCommitTime = metaClient.reloadActiveTimeline().getCompletedReplaceTimeline()
- .getReverseOrderedInstants().findFirst().get().getTimestamp();
- verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(), clusterMetadata.getWriteStatuses(), config);
- }
- }
-
- private HoodieWriteMetadata<List<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
- boolean populateMetaFields,
- boolean completeClustering,
- String validatorClasses,
- String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
- Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException {
- HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
- .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
- .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
- .build();
-
- HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
- .withPreCommitValidatorConfig(validatorConfig)
- .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
- .withClusteringConfig(clusteringConfig).build();
-
- // create client with new config.
- HoodieJavaWriteClient client = getHoodieWriteClient(config);
- String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
- HoodieWriteMetadata<List<WriteStatus>> clusterMetadata = client.cluster(clusteringCommitTime, completeClustering);
- if (config.populateMetaFields()) {
- verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses());
- } else {
- verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses(), config);
- }
-
- Set<HoodieFileGroupId> replacedFileIds = new HashSet<>();
- clusterMetadata.getPartitionToReplaceFileIds().entrySet().forEach(partitionFiles ->
- partitionFiles.getValue().stream().forEach(file ->
- replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file))));
- return clusterMetadata;
- }
-
- private Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
- return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
- }
-
- private Set<String> insertPartitionRecordsWithCommit(HoodieJavaWriteClient client, int recordsCount, String commitTime1, String partitionPath) throws IOException {
- client.startCommitWithTime(commitTime1);
- List<HoodieRecord> inserts1 = dataGen.generateInsertsForPartition(commitTime1, recordsCount, partitionPath);
- List<WriteStatus> statuses = client.upsert(inserts1, commitTime1);
- assertNoWriteErrors(statuses);
- Set<String> batchBuckets = statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
- verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig());
- return batchBuckets;
- }
-
- /**
- * Verify data in base files matches expected records and commit time.
- */
- private void verifyRecordsWritten(String commitTime, boolean populateMetadataField,
- List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, HoodieWriteConfig config) throws IOException {
- List<GenericRecord> records = new ArrayList<>();
- Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
- if (config.populateMetaFields()) {
- for (GenericRecord record : records) {
- String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- assertEquals(commitTime,
- record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
- assertTrue(expectedKeys.contains(recordKey));
- }
- } else {
- KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()));
- for (GenericRecord record : records) {
- String recordKey = keyGenerator.getKey(record).getRecordKey();
- if (!populateMetadataField) {
- assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
- }
- assertTrue(expectedKeys.contains(recordKey));
- }
- }
- }
-
- @NotNull
- private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
- for (WriteStatus status : allStatus) {
- StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
- }
- Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
- assertEquals(records.size(), expectedKeys.size());
- return expectedKeys;
- }
-
- private void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
- List<GenericRecord> records = new ArrayList<>();
- Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
- Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
- .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
- assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
- Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
- for (GenericRecord record : records) {
- String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- assertTrue(expectedKeys.contains(recordKey));
- String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
- assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
- }
- }
-
- private List<WriteStatus> writeAndVerifyBatch(HoodieJavaWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
- return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
- }
-
- private List<WriteStatus> writeAndVerifyBatch(HoodieJavaWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
- client.startCommitWithTime(commitTime);
- List<WriteStatus> statusRDD = client.upsert(inserts, commitTime);
- if (autoCommitOff) {
- client.commit(commitTime, statusRDD);
- }
- List<WriteStatus> statuses = statusRDD;
- assertNoWriteErrors(statuses);
- verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig());
-
- return statuses;
- }
-
- private Pair<Set<String>, List<HoodieRecord>> testUpdates(String instantTime, HoodieJavaWriteClient client,
- int sizeToInsertAndUpdate, int expectedTotalRecords)
- throws IOException {
- client.startCommitWithTime(instantTime);
- List<HoodieRecord> inserts = dataGen.generateInserts(instantTime, sizeToInsertAndUpdate);
- Set<String> keys = recordsToRecordKeySet(inserts);
- List<HoodieRecord> insertsAndUpdates = new ArrayList<>();
- insertsAndUpdates.addAll(inserts);
- insertsAndUpdates.addAll(dataGen.generateUpdates(instantTime, inserts));
-
- List<WriteStatus> statuses = client.upsert(insertsAndUpdates, instantTime);
- assertNoWriteErrors(statuses);
-
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(expectedTotalRecords,
- countRowsInPaths(basePath, storage, fullPartitionPaths),
- "Must contain " + expectedTotalRecords + " records");
- return Pair.of(keys, inserts);
+ testFailWritesOnInlineTableServiceExceptions(shouldFail, createBrokenClusteringClient);
}
/**
* Test delete with delete api.
*/
- @Test
- public void testDeletesWithoutInserts() {
- final String testPartitionPath = "2016/09/26";
- final int insertSplitLimit = 100;
- // setup the small file handling params
- HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
- TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), true, true
- ? new Properties() : getPropertiesForKeyGen());
- dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
- HoodieJavaWriteClient client = getHoodieWriteClient(config);
-
- // delete non existent keys
- String commitTime1 = "001";
- client.startCommitWithTime(commitTime1);
-
- List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
- List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
- client.delete(hoodieKeysToDelete, commitTime1);
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDeletesWithoutInserts(boolean populateMetaFields) {
+ testDeletesWithoutInserts(populateMetaFields, IDENTITY, IDENTITY);
}
/**
@@ -1116,104 +379,16 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
*/
@Test
public void testCommitWritesRelativePaths() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
- HoodieTableMetaClient metaClient = createMetaClient();
- HoodieJavaTable table = HoodieJavaTable.create(cfgBuilder.build(), context, metaClient);
-
- String instantTime = "000";
- client.startCommitWithTime(instantTime);
-
- List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
-
- List<WriteStatus> result = client.bulkInsert(records, instantTime);
-
- assertTrue(client.commit(instantTime, result), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
-
- // Get base file paths from commit metadata
- String actionType = metaClient.getCommitActionType();
- HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
- HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(),
- HoodieCommitMetadata.class);
- String basePath = table.getMetaClient().getBasePath();
- Collection<String> commitPathNames =
- commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
-
- // Read from commit file
- HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline()
- .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
- HoodieCommitMetadata.class);
- HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
- // Compare values in both to make sure they are equal.
- for (String pathName : paths.values()) {
- assertTrue(commitPathNames.contains(pathName));
- }
- }
+ testCommitWritesRelativePaths(IDENTITY);
}
/**
* Test to ensure commit metadata points to valid files.10.
*/
- @Test
- public void testMetadataStatsOnCommit() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- HoodieWriteConfig cfg = cfgBuilder.build();
- HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-
- String instantTime0 = "000";
- client.startCommitWithTime(instantTime0);
-
- List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
- List<WriteStatus> result0 = client.bulkInsert(records0, instantTime0);
-
- assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime0),
- "After explicit commit, commit file should be created");
-
- // Read from commit file
- HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
- HoodieCommitMetadata.class);
- int inserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
- }
- }
- assertEquals(200, inserts);
-
- // Update + Inserts such that they just expand file1
- String instantTime1 = "001";
- client.startCommitWithTime(instantTime1);
-
- List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
- List<WriteStatus> result1 = client.upsert(records1, instantTime1);
-
- assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime1),
- "After explicit commit, commit file should be created");
-
- // Read from commit file
- metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
- HoodieCommitMetadata.class);
- inserts = 0;
- int upserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
- upserts += stat.getNumUpdateWrites();
- }
- }
- assertEquals(0, inserts);
- assertEquals(200, upserts);
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testMetadataStatsOnCommit(boolean populateMetaFields) throws Exception {
+ testMetadataStatsOnCommit(populateMetaFields, IDENTITY);
}
/**
@@ -1222,71 +397,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
- HoodieTableMetaClient metaClient = createMetaClient();
- String instantTime = "000";
- HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
- HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
- Pair<StoragePath, List<WriteStatus>> result = testConsistencyCheck(
- metaClient, instantTime, enableOptimisticConsistencyGuard);
-
- // Delete orphan marker and commit should succeed
- metaClient.getStorage().deleteFile(result.getKey());
- if (!enableOptimisticConsistencyGuard) {
- assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
- // Marker directory must be removed
- assertFalse(metaClient.getStorage()
- .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- } else {
- // with optimistic, first client.commit should have succeeded.
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
- // Marker directory must be removed
- assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- }
+ testConsistencyCheckDuringFinalize(context, enableOptimisticConsistencyGuard, getHoodieTable, IDENTITY, IDENTITY);
}
private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
boolean populateMetaFields) throws Exception {
- String instantTime = "00000000000010";
- HoodieTableMetaClient metaClient = createMetaClient();
-
- Properties properties = new Properties();
- if (!populateMetaFields) {
- properties = getPropertiesForKeyGen();
- }
-
- HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build() :
- getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
- .withConsistencyCheckEnabled(true)
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
- .withOptimisticConsistencyGuardSleepTimeMs(1).build())
- .withProperties(properties).build();
- HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
- testConsistencyCheck(metaClient, instantTime, enableOptimisticConsistencyGuard);
-
- if (!enableOptimisticConsistencyGuard) {
- // Rollback of this commit should succeed with FailSafeCG
- client.rollback(instantTime);
- assertFalse(testTable.commitExists(instantTime),
- "After explicit rollback, commit file should not be present");
- // Marker directory must be removed after rollback
- assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- } else {
- // if optimistic CG is enabled, commit should have succeeded.
- assertTrue(testTable.commitExists(instantTime),
- "With optimistic CG, first commit should succeed. commit file should be present");
- // Marker directory must be removed after rollback
- assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- client.rollback(instantTime);
- assertFalse(testTable.commitExists(instantTime),
- "After explicit rollback, commit file should not be present");
- }
+ testRollbackAfterConsistencyCheckFailureUsingFileList(context, rollbackUsingMarkers, enableOptimisticConsistencyGuard,
+ populateMetaFields, getHoodieTable, IDENTITY, IDENTITY);
}
@ParameterizedTest
@@ -1303,303 +420,17 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
@Test
public void testRollbackFailedCommits() throws Exception {
- // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
- HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
- boolean populateMetaFields = true;
- HoodieTestUtils.init(storageConf, basePath);
- HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-
- // perform 1 successful commit
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, true);
-
- // Perform 2 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // refresh data generator to delete records generated from failed commits
- dataGen = new HoodieTestDataGenerator();
- // Perform 1 successful write
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, true);
- HoodieTableMetaClient metaClient = createMetaClient();
-
- assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
- assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
- assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
- // Await till enough time passes such that the first 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
- Thread.sleep(2000);
- }
- client.close();
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- // Perform 1 successful write
- writeBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, true);
- client.clean();
- client.close();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- if (cleaningPolicy.isLazy()) {
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
- .countInstants()
- == 2);
- // Since we write rollbacks not clean, there should be no clean action on the timeline
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
- .countInstants()
- == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- } else if (cleaningPolicy.isNever()) {
- // never will get translated to Lazy if OCC is enabled.
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
- .countInstants()
- == 2);
- // There should be no clean or rollback action on the timeline
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
- .countInstants()
- == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- }
+ super.testRollbackFailedCommits(true);
}
@Test
public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception {
- HoodieTestUtils.init(storageConf, basePath);
- HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
- HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // Perform 1 successful writes to table
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, true);
-
- // Perform 1 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Toggle cleaning policy to LAZY
- cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
- // Perform 2 failed writes to table
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Await till enough time passes such that the 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
- Thread.sleep(2000);
- }
- client.clean();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
- // Perform 2 failed commits
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Toggle cleaning policy to EAGER
- cleaningPolicy = EAGER;
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- client.startCommit();
- timeline = metaClient.getActiveTimeline().reload();
- // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1);
- client.close();
+ super.testRollbackFailedCommitsToggleCleaningPolicy(true);
}
@Test
public void testParallelInsertAndCleanPreviousFailedCommits() throws Exception {
- HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
- ExecutorService service = Executors.newFixedThreadPool(2);
- HoodieTestUtils.init(storageConf, basePath);
- HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // perform 1 successful write
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
- 0, true);
-
- // Perform 2 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
- 0, false);
- client.close();
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
- 0, false);
- client.close();
- // refresh data generator to delete records generated from failed commits
- dataGen = new HoodieTestDataGenerator();
- // Create a successful commit
- Future<List<WriteStatus>> commit3 = service.submit(() -> writeBatch(new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
- "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
- HoodieJavaWriteClient::bulkInsert, false, 100, 100, 0, true));
- commit3.get();
- HoodieTableMetaClient metaClient = createMetaClient();
-
- assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
- assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
- assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
- client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // Await till enough time passes such that the first 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
- Thread.sleep(2000);
- }
- Future<List<WriteStatus>> commit4 = service.submit(() -> writeBatch(new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
- "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
- HoodieJavaWriteClient::bulkInsert, false, 100, 100, 0, true));
- Future<HoodieCleanMetadata> clean1 = service.submit(() -> new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)).clean());
- commit4.get();
- clean1.get();
- client.close();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 2);
- // Since we write rollbacks not clean, there should be no clean action on the timeline
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(CLEAN_ACTION)).countInstants() == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- service.shutdown();
- }
-
- private Pair<StoragePath, List<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
- throws Exception {
- HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
- .build()) : (getConfigBuilder().withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
- .withOptimisticConsistencyGuardSleepTimeMs(1).build())
- .build());
- HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-
- client.startCommitWithTime(instantTime);
- List<HoodieRecord> writeRecords = dataGen.generateInserts(instantTime, 200);
- List<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-
- // Create a dummy marker file to simulate the case that a marker file was created without data file.
- // This should fail the commit
- String partitionPath;
- String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
- FileSystem fs = (FileSystem) storage.getFileSystem();
- if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
- String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
- markerFolderPath, storage, context, 1).values().stream()
- .flatMap(Collection::stream).findFirst().get();
- partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
- } else {
- partitionPath = Arrays
- .stream(fs.globStatus(new Path(String.format("%s/*/*/*/*", markerFolderPath)),
- path -> path.toString().contains(HoodieTableMetaClient.MARKER_EXTN)))
- .limit(1).map(status -> status.getPath().getParent().toString()).collect(Collectors.toList()).get(0);
- }
-
- Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
- cfg.getMarkersType(), getHoodieTable(metaClient, cfg), instantTime)
- .create(partitionPath,
- FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()),
- IOType.MERGE);
- if (!enableOptimisticConsistencyGuard) {
- Exception e = assertThrows(HoodieCommitException.class, () -> {
- client.commit(instantTime, result);
- }, "Commit should fail due to consistency check");
- assertTrue(e.getCause() instanceof HoodieIOException);
- } else {
- // with optimistic CG, commit should succeed
- client.commit(instantTime, result);
- }
- return Pair.of(markerFilePath.get(), result);
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
- return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
- boolean populateMetaFields, Properties props) {
- HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
- if (!populateMetaFields) {
- builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.SIMPLE).build());
- }
- return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
- .compactionSmallFileSize(smallFileSize)
- // Set rollback to LAZY so no inflights are deleted
- .insertSplitSize(insertSplitSize).build())
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
- .build())
- .withStorageConfig(HoodieStorageConfig.newBuilder()
- .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
- .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
- .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
- .withProps(props)
- .build();
- }
-
- private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
- Properties properties = new Properties();
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
- if (!populateMetaFields) {
- getPropertiesForKeyGen(populateMetaFields).entrySet().forEach(kv ->
- properties.put(kv.getKey(), kv.getValue()));
- }
- return getConfigBuilder()
- .withEmbeddedTimelineServerEnabled(false)
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(cleaningPolicy)
- .withAutoClean(false).build())
- .withTimelineLayoutVersion(1)
- .withHeartbeatIntervalInMs(3 * 1000)
- .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
- .withRemoteServerPort(timelineServicePort).build())
- .withAutoCommit(false)
- .withLockConfig(HoodieLockConfig.newBuilder()
- .withLockProvider(InProcessLockProvider.class)
- .build())
- .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
- .withProperties(properties).build();
+ super.testParallelInsertAndCleanPreviousFailedCommits(true);
}
public static class WriteClientBrokenClustering<T extends HoodieRecordPayload> extends org.apache.hudi.client.HoodieJavaWriteClient<T> {
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index 520e5ba0a1f..7aac330a99d 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -17,6 +17,7 @@
package org.apache.hudi.testutils;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.client.HoodieJavaWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieJavaEngineContext;
@@ -50,7 +51,6 @@ import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.util.FileFormatUtils;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
@@ -63,6 +63,7 @@ import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.JavaHoodieIndexFactory;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.HoodieTableMetadata;
@@ -106,6 +107,8 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.testutils.HoodieTestUtils.RAW_TRIPS_TEST_NAME;
import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForKeys;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForRecords;
import static org.apache.hudi.testutils.GenericRecordValidationTestUtils.readHFile;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -121,10 +124,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
private static final Logger LOG = LoggerFactory.getLogger(HoodieJavaClientTestHarness.class);
- protected StorageConfiguration<Configuration> storageConf;
protected HoodieJavaEngineContext context;
protected TestJavaTaskContextSupplier taskContextSupplier;
- protected HoodieStorage storage;
protected ExecutorService executorService;
protected HoodieTableFileSystemView tableView;
protected HoodieJavaWriteClient writeClient;
@@ -239,6 +240,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
}
+ @Override
protected HoodieJavaWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
if (null != writeClient) {
writeClient.close();
@@ -264,6 +266,20 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
return HoodieTableMetadata.create(engineContext, metaClient.getStorage(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
+ @Override
+ protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
+ client.startCommitWithTime(commitTime);
+ List<WriteStatus> statusRDD = ((HoodieJavaWriteClient) client).upsert(inserts, commitTime);
+ if (autoCommitOff) {
+ client.commit(commitTime, statusRDD);
+ }
+ List<WriteStatus> statuses = statusRDD;
+ assertNoWriteErrors(statuses);
+ verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig(),
+ HoodieAvroKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
+ return statuses;
+ }
+
/**
* Validate the metadata tables contents to ensure it matches what is on the file system.
*/
@@ -739,6 +755,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
* @param wrapped Actual Records Generation function
* @return Wrapped Function
*/
+ @Override
public Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI,
HoodieWriteConfig writeConfig,
Function2<List<HoodieRecord>, String, Integer> wrapped) {
@@ -1027,12 +1044,4 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
return builder;
}
- public static FileFormatUtils getFileUtilsInstance(HoodieTableMetaClient metaClient) {
- return HoodieIOFactory.getIOFactory(metaClient.getStorage())
- .getFileFormatUtils(metaClient.getTableConfig().getBaseFileFormat());
- }
-
- protected HoodieTableMetaClient createMetaClient() {
- return HoodieTestUtils.createMetaClient(storageConf, basePath);
- }
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
index 9ed2dce3ce5..f5fa70c6668 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
@@ -265,7 +265,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase {
// Initial inserts with TRIP_EXAMPLE_SCHEMA
int numRecords = 10;
insertFirstBatch(hoodieWriteConfig, client, "001", initCommitTime,
- numRecords, SparkRDDWriteClient::insert, false, true, numRecords);
+ numRecords, SparkRDDWriteClient::insert, false, true, numRecords);
checkReadRecords("000", numRecords);
// Updates with same schema is allowed
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 35ffd1035e1..b73ed53e62f 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -2285,7 +2285,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
context = new HoodieSparkEngineContext(jsc);
- HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+ HoodieWriteConfig config = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
SparkRDDWriteClient client = getHoodieWriteClient(config);
// Write 1 (Bulk insert)
@@ -2356,7 +2356,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
context = new HoodieSparkEngineContext(jsc);
- HoodieWriteConfig initialConfig = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+ HoodieWriteConfig initialConfig = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
.withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build()).build();
SparkRDDWriteClient client = getHoodieWriteClient(config);
@@ -3278,23 +3278,6 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
}
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
- HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
- return builder.withCompactionConfig(
- HoodieCompactionConfig.newBuilder()
- .compactionSmallFileSize(smallFileSize)
- // Set rollback to LAZY so no inflights are deleted
- .insertSplitSize(insertSplitSize).build())
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
- .withStorageConfig(
- HoodieStorageConfig.newBuilder()
- .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
- .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
- .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
- .build();
- }
-
public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, HoodieIndex.IndexType indexType,
HoodieFailedWritesCleaningPolicy cleaningPolicy) {
Properties properties = getDisabledRowWriterProperties();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 335a0f8f86a..834616ee804 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -18,7 +18,6 @@
package org.apache.hudi.client.functional;
-import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.client.BaseHoodieWriteClient;
@@ -29,104 +28,74 @@ import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.clustering.plan.strategy.SparkSingleFileSortPlanStrategy;
import org.apache.hudi.client.clustering.run.strategy.SparkSingleFileSortExecutionStrategy;
import org.apache.hudi.client.clustering.update.strategy.SparkRejectUpdateStrategy;
+import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
import org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass;
import org.apache.hudi.client.transaction.PreferWriterConflictResolutionStrategy;
import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy;
-import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
import org.apache.hudi.client.validator.SparkPreCommitValidator;
import org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator;
import org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator;
import org.apache.hudi.common.config.HoodieMetadataConfig;
-import org.apache.hudi.common.config.HoodieStorageConfig;
-import org.apache.hudi.common.config.LockConfiguration;
-import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.FileSlice;
-import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodieOperation;
-import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordDelegate;
import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.HoodieWriteStat;
-import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.marker.MarkerType;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestTable;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.testutils.RawTripTestPayload;
import org.apache.hudi.common.util.ClusteringUtils;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FileFormatUtils;
-import org.apache.hudi.common.util.MarkerUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieClusteringConfig;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.exception.HoodieClusteringException;
-import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieCorruptedDataException;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.exception.HoodieValidationException;
import org.apache.hudi.exception.HoodieWriteConflictException;
import org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner;
-import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.index.HoodieIndex.IndexType;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieSparkCopyOnWriteTable;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.cluster.ClusteringTestUtils;
-import org.apache.hudi.table.action.commit.HoodieWriteHelper;
-import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.hudi.testutils.HoodieClientTestBase;
import org.apache.hudi.testutils.HoodieClientTestUtils;
import org.apache.hudi.testutils.HoodieSparkWriteableTestTable;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.Path;
+import org.apache.spark.api.java.AbstractJavaRDDLike;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -138,7 +107,6 @@ import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -146,27 +114,19 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY;
import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
-import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
@@ -174,26 +134,22 @@ import static org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENA
import static org.apache.hudi.config.HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME;
import static org.apache.hudi.config.HoodieClusteringConfig.UPDATES_STRATEGY;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadata;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
@SuppressWarnings("unchecked")
@Tag("functional")
public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
- private static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
private static final Map<String, String> STRATEGY_PARAMS = new HashMap<String, String>() {
{
put("sortColumn", "record_key");
}
};
-
private static Stream<Arguments> smallInsertHandlingParams() {
return Arrays.stream(new Boolean[][] {{true}, {false}}).map(Arguments::of);
}
@@ -202,14 +158,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
return Arrays.stream(new Boolean[][] {{true}, {false}}).map(Arguments::of);
}
- private static Stream<Arguments> rollbackFailedCommitsParams() {
- return Stream.of(
- Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, true),
- Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, true),
- Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, false)
- );
- }
-
private static Stream<Arguments> rollbackAfterConsistencyCheckFailureParams() {
return Stream.of(
Arguments.of(true, true),
@@ -219,22 +167,102 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
);
}
- private HoodieTestTable testTable;
-
private static final String COUNT_SQL_QUERY_FOR_VALIDATION = "select count(*) from <TABLE_NAME>";
+ private static HoodiePreCommitValidatorConfig createPreCommitValidatorConfig(int expectedNumberOfRows) {
+ return HoodiePreCommitValidatorConfig.newBuilder()
+ .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
+ .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + expectedNumberOfRows)
+ .build();
+ }
+
+ private static HoodieClusteringConfig.Builder createClusteringBuilder(boolean isInline, int inlineNumCommits) {
+ return HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0)
+ .withInlineClustering(isInline).withInlineClusteringNumCommits(inlineNumCommits)
+ .fromProperties(getDisabledRowWriterProperties());
+ }
+
+ private static HoodieLockConfig createLockConfig(ConflictResolutionStrategy conflictResolutionStrategy) {
+ return HoodieLockConfig.newBuilder().withLockProvider(FileSystemBasedLockProviderTestClass.class)
+ .withConflictResolutionStrategy(conflictResolutionStrategy).build();
+ }
+
+ private static HoodieCleanConfig createCleanConfig(HoodieFailedWritesCleaningPolicy policy, boolean autoClean) {
+ return HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(policy).withAutoClean(autoClean).build();
+ }
+
@BeforeEach
public void setUpTestTable() {
testTable = HoodieSparkWriteableTestTable.of(metaClient);
}
+ private final Function<List, JavaRDD> list2Rdd = recordList -> jsc.parallelize(recordList, 1);
+
+ private final Function<JavaRDD, List> rdd2List = AbstractJavaRDDLike::collect;
+
+ private final Function<HoodieWriteConfig, BaseHoodieWriteClient> createBrokenClusteringClient =
+ config -> new WriteClientBrokenClustering<>(context, config);
+
+ private final Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> clusteringMetadataRdd2List =
+ metadata -> metadata.clone(((JavaRDD)(metadata.getWriteStatuses())).collect());
+
+ private final Function<HoodieWriteConfig, KeyGenerator> createKeyGenerator =
+ config -> HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps());
+
+ private final Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTable =
+ (metaClient, config) -> getHoodieTable(metaClient, config);
+
+ @Override
+ protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+ return insertFirstBatch(writeConfig, (SparkRDDWriteClient) client, newCommitTime, initCommitTime, numRecordsInThisCommit,
+ (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+ isPreppedAPI, assertForCommit, expRecordsInThisCommit, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ return writeBatch((SparkRDDWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+ (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+ assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+ Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+ Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+ boolean filterForCommitTimeWithAssert) throws Exception {
+ final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+ generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates);
+
+ return writeBatch((SparkRDDWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+ (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime), assertForCommit, expRecordsInThisCommit, expTotalRecords,
+ expTotalCommits, false, filterForCommitTimeWithAssert);
+ }
+
+ @Override
+ protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+ String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+ boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+ return deleteBatch(writeConfig, (SparkRDDWriteClient) client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit,
+ isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, filterForCommitTimeWithAssert);
+ }
+
/**
* Test Auto Commit behavior for HoodieWriteClient insert API.
*/
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testAutoCommitOnInsert(boolean populateMetaFields) throws Exception {
- testAutoCommit(SparkRDDWriteClient::insert, false, populateMetaFields);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insert(recordRDD, instantTime), false, populateMetaFields);
}
/**
@@ -242,7 +270,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testAutoCommitOnInsertPrepped() throws Exception {
- testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true, true);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insertPreppedRecords(recordRDD, instantTime), true, true);
}
/**
@@ -251,7 +279,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testAutoCommitOnUpsert(boolean populateMetaFields) throws Exception {
- testAutoCommit(SparkRDDWriteClient::upsert, false, populateMetaFields);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), false, populateMetaFields);
}
/**
@@ -260,7 +288,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testAutoCommitOnUpsertPrepped(boolean populateMetaFields) throws Exception {
- testAutoCommit(SparkRDDWriteClient::upsertPreppedRecords, true, populateMetaFields);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, populateMetaFields);
}
/**
@@ -269,7 +297,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testAutoCommitOnBulkInsert(boolean populateMetaFields) throws Exception {
- testAutoCommit(SparkRDDWriteClient::bulkInsert, false, populateMetaFields);
+ testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsert(recordRDD, instantTime), false, populateMetaFields);
}
/**
@@ -281,42 +309,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
Option.empty()), true, true);
}
- /**
- * Test auto-commit by applying the write function.
- *
- * @param writeFn One of HoodieWriteClient Write API
- * @throws Exception in case of failure
- */
- private void testAutoCommit(Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn,
- boolean isPrepped, boolean populateMetaFields) throws Exception {
- // Set autoCommit false
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
- try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());) {
-
- String prevCommitTime = "000";
- String newCommitTime = "001";
- int numRecords = 200;
- JavaRDD<WriteStatus> result = insertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn,
- isPrepped, false, numRecords);
-
- assertFalse(testTable.commitExists(newCommitTime),
- "If Autocommit is false, then commit should not be made automatically");
- assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
- assertTrue(testTable.commitExists(newCommitTime),
- "After explicit commit, commit file should be created");
- }
- }
-
@Test
public void testPreCommitValidatorsOnInsert() throws Exception {
int numRecords = 200;
- HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + numRecords)
- .build();
HoodieWriteConfig config = getConfigBuilder().withAutoCommit(true)
- .withPreCommitValidatorConfig(validatorConfig).build();
+ .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200)).build();
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
writeClient.bulkInsert(recordRDD, instantTime, Option.empty());
@@ -330,12 +327,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testPreCommitValidationFailureOnInsert() throws Exception {
int numRecords = 200;
- HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
- //set wrong value for expected number of rows
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + 500)
- .build();
- HoodieWriteConfig config = getConfigBuilder().withPreCommitValidatorConfig(validatorConfig).build();
+ //set wrong value for expected number of rows
+ HoodieWriteConfig config = getConfigBuilder().withPreCommitValidatorConfig(createPreCommitValidatorConfig(500)).build();
String newCommitTime = metaClient.createNewInstantTime();
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
@@ -357,14 +350,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testPreCommitValidationWithMultipleInflights() throws Exception {
int numRecords = 200;
- HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
- //set wrong value for expected number of rows
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + 500)
- .build();
HoodieWriteConfig config = getConfigBuilder()
- .withCleanConfig(HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.NEVER).build())
- .withPreCommitValidatorConfig(validatorConfig)
+ .withCleanConfig(createCleanConfig(HoodieFailedWritesCleaningPolicy.NEVER, true))
+ //set wrong value for expected number of rows
+ .withPreCommitValidatorConfig(createPreCommitValidatorConfig(500))
.build();
String instant1 = getHoodieWriteClient(config).createNewInstantTime();
@@ -383,14 +372,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertTrue(testTable.inflightCommitExists(instant1));
numRecords = 300;
- validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
- //set wrong value for expected number of rows
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + numRecords)
- .build();
config = getConfigBuilder()
- .withCleanConfig(HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.NEVER).build())
- .withPreCommitValidatorConfig(validatorConfig)
+ .withCleanConfig(createCleanConfig(HoodieFailedWritesCleaningPolicy.NEVER, true))
+ .withPreCommitValidatorConfig(createPreCommitValidatorConfig(numRecords))
.build();
String instant2 = getHoodieWriteClient(config).createNewInstantTime();
// expect pre-commit validators to succeed. Note that validator is expected to exclude inflight instant1
@@ -414,7 +398,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testDeduplicationOnInsert(boolean populateMetaFields) throws Exception {
- testDeduplication(SparkRDDWriteClient::insert, populateMetaFields);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.insert(list2Rdd.apply(records), commitTime)), populateMetaFields, false);
}
/**
@@ -423,7 +407,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testDeduplicationKeepOperationFieldOnInsert(boolean populateMetaFields) throws Exception {
- testDeduplicationKeepOperation(SparkRDDWriteClient::insert, populateMetaFields);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.insert(list2Rdd.apply(records), commitTime)), populateMetaFields, true);
}
/**
@@ -432,7 +416,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testDeduplicationOnBulkInsert(boolean populateMetaFields) throws Exception {
- testDeduplication(SparkRDDWriteClient::bulkInsert, populateMetaFields);
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.bulkInsert(list2Rdd.apply(records), commitTime)), populateMetaFields, false);
}
/**
@@ -440,145 +424,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testDeduplicationOnUpsert() throws Exception {
- testDeduplication(SparkRDDWriteClient::upsert, true);
- }
-
- /**
- * Test Deduplication Logic for write function.
- *
- * @param writeFn One of HoodieWriteClient non-prepped write APIs
- * @throws Exception in case of failure
- */
- private void testDeduplication(
- Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
- String newCommitTime = "001";
-
- String recordKey = UUID.randomUUID().toString();
- HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
- HoodieRecord<RawTripTestPayload> recordOne =
- new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime));
-
- HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
- HoodieRecord recordTwo =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
- // Same key and partition as keyTwo
- HoodieRecord recordThree =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
- HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieJavaRDD.of(
- jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1));
- HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
- .combineInput(true, true);
- addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
- HoodieWriteConfig writeConfig = configBuilder.build();
-
- // Global dedup should be done based on recordKey only
- HoodieIndex index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(true);
- int dedupParallelism = records.getNumPartitions() + 2;
- HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(dedupParallelism, dedupedRecsRdd.getNumPartitions());
- assertEquals(1, dedupedRecs.size());
- assertEquals(dedupedRecs.get(0).getPartitionPath(), recordThree.getPartitionPath());
- assertNodupesWithinPartition(dedupedRecs);
-
- // non-Global dedup should be done based on both recordKey and partitionPath
- index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(false);
- dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(dedupParallelism, dedupedRecsRdd.getNumPartitions());
- assertEquals(2, dedupedRecs.size());
- assertNodupesWithinPartition(dedupedRecs);
-
- // Perform write-action and check
- JavaRDD<HoodieRecord> recordList = jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1);
-
- try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
- client.startCommitWithTime(newCommitTime);
- List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime).collect();
- assertNoWriteErrors(statuses);
- assertEquals(2, statuses.size());
- assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
- .collect(Collectors.toList()));
- }
- }
-
- /**
- * Test Deduplication Logic for write function.
- *
- * @param writeFn One of HoodieWriteClient non-prepped write APIs
- * @throws Exception in case of failure
- */
- private void testDeduplicationKeepOperation(
- Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
- String newCommitTime = "001";
-
- String recordKey = UUID.randomUUID().toString();
- HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
- HoodieRecord<RawTripTestPayload> recordOne =
- new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime), HoodieOperation.INSERT);
-
- HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
- HoodieRecord recordTwo =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.INSERT);
-
- // Same key and partition as keyTwo
- HoodieRecord recordThree =
- new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.UPDATE_AFTER);
-
- HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieJavaRDD.of(
- jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1));
- HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAllowOperationMetadataField(true)
- .combineInput(true, true);
- addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
- HoodieWriteConfig writeConfig = configBuilder.build();
-
- // Global dedup should be done based on recordKey only
- HoodieIndex index = mock(HoodieIndex.class);
- when(index.isGlobal()).thenReturn(true);
- int dedupParallelism = records.getNumPartitions() + 100;
- HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
- (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
- .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
- List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
- assertEquals(dedupedRecs.get(0).getOperation(), recordThree.getOperation());
-
- // Perform write-action and check
- JavaRDD<HoodieRecord> recordList = jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1);
-
- try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
- client.startCommitWithTime(newCommitTime);
- List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime).collect();
- assertNoWriteErrors(statuses);
- assertEquals(2, statuses.size());
- assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
- .collect(Collectors.toList()));
- }
- }
-
- /**
- * Assert that there is no duplicate key at the partition level.
- *
- * @param recordDelegates List of Hoodie record delegates
- */
- void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
- Map<String, Set<String>> partitionToKeys = new HashMap<>();
- for (HoodieRecordDelegate r : recordDelegates) {
- String recordKey = r.getRecordKey();
- String partitionPath = r.getPartitionPath();
- if (!partitionToKeys.containsKey(partitionPath)) {
- partitionToKeys.put(partitionPath, new HashSet<>());
- }
- assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
- partitionToKeys.get(partitionPath).add(recordKey);
- }
+ testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.upsert(list2Rdd.apply(records), commitTime)), true, false);
}
/**
@@ -587,9 +433,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testUpserts(boolean populateMetaFields) throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
- testUpsertsInternal(cfgBuilder.build(), SparkRDDWriteClient::upsert, false);
+ testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), populateMetaFields, false);
}
/**
@@ -597,132 +441,27 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testUpsertsPrepped() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testUpsertsInternal(cfgBuilder.build(), SparkRDDWriteClient::upsertPreppedRecords, true);
+ testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
}
- /**
- * Test one of HoodieWriteClient upsert(Prepped) APIs.
- *
- * @param config Write Config
- * @param writeFn One of Hoodie Write Function API
- * @throws Exception in case of error
- */
- private void testUpsertsInternal(HoodieWriteConfig config,
- Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean isPrepped)
- throws Exception {
- // Force using older timeline layout
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
- .withRollbackUsingMarkers(true)
- .withProps(config.getProps()).withTimelineLayoutVersion(
- VERSION_0).build();
-
- HoodieTableMetaClient.withPropertyBuilder()
- .fromMetaClient(metaClient)
- .setTimelineLayoutVersion(VERSION_0)
- .setPopulateMetaFields(config.populateMetaFields())
- .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
- SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String newCommitTime = "001";
- String initCommitTime = "000";
- int numRecords = 200;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert,
- isPrepped, true, numRecords, config.populateMetaFields());
-
- // Write 2 (updates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- numRecords = 100;
- String commitTimeBetweenPrevAndNew = "002";
- updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime,
- Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
- numRecords, 200, 2, config.populateMetaFields());
-
- // Delete 1
- prevCommitTime = newCommitTime;
- newCommitTime = "005";
- numRecords = 50;
-
- deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
- 0, 150, config.populateMetaFields());
-
- // Now simulate an upgrade and perform a restore operation
- HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
- TimelineLayoutVersion.CURR_VERSION).build();
- client = getHoodieWriteClient(newConfig);
-
- client.savepoint("004", "user1", "comment1");
-
- client.restoreToInstant("004", config.isMetadataTableEnabled());
-
- assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
-
- client.deleteSavepoint("004");
- assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
-
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(200,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + 200 + " records");
-
- // Perform Delete again on upgraded dataset.
- prevCommitTime = newCommitTime;
- newCommitTime = "006";
- numRecords = 50;
-
- deleteBatch(newConfig, client, newCommitTime, prevCommitTime,
- initCommitTime, numRecords, isPrepped, true,
- 0, 150);
-
- HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
- List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
- assertEquals(5, instants.size());
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"),
- instants.get(0));
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"),
- instants.get(1));
- // New Format should have all states of instants
- assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"),
- instants.get(2));
- assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"),
- instants.get(3));
- assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"),
- instants.get(4));
-
- final HoodieWriteConfig cfg = hoodieWriteConfig;
+ @Override
+ protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
final String instantTime = "007";
HoodieTableMetaClient metaClient = HoodieClientTestUtils.createMetaClient(jsc, basePath);
- String basePathStr = basePath;
- HoodieTable table = getHoodieTable(metaClient, cfg);
- String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
+ HoodieTable table = getHoodieTable(metaClient, config);
+ Pair<String, String> partitionAndBaseFilePaths = getPartitionAndBaseFilePathsFromLatestCommitMetadata(metaClient);
+ String partitionPath = partitionAndBaseFilePaths.getLeft();
+ String baseFilePath = partitionAndBaseFilePaths.getRight();
jsc.parallelize(Arrays.asList(1)).map(e -> {
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
- metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
- HoodieCommitMetadata.class);
- String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
- .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
- .map(ee -> ee.getPath()).orElse(null);
- String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
- .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
- .map(ee -> ee.getPartitionPath()).orElse(null);
- Path baseFilePath = new Path(basePathStr, filePath);
- HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath.toString());
+
+ HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath);
HoodieMergeHandle handle = null;
try {
- handle = new HoodieMergeHandle(cfg, instantTime, table, new HashMap<>(),
- partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new SparkTaskContextSupplier(),
+ handle = new HoodieMergeHandle(config, instantTime, table, new HashMap<>(),
+ partitionPath, FSUtils.getFileId(baseFile.getFileName()), baseFile, new SparkTaskContextSupplier(),
config.populateMetaFields() ? Option.empty() :
- Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+ Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps())));
WriteStatus writeStatus = new WriteStatus(false, 0.0);
writeStatus.setStat(new HoodieWriteStat());
writeStatus.getStat().setNumWrites(0);
@@ -738,12 +477,12 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
handle = null;
try {
final String newInstantTime = "006";
- cfg.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
- HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(cfg.getProps()).build();
+ config.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
+ HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(config.getProps()).build();
handle = new HoodieMergeHandle(cfg2, newInstantTime, table, new HashMap<>(),
- partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new SparkTaskContextSupplier(),
+ partitionPath, FSUtils.getFileId(baseFile.getFileName()), baseFile, new SparkTaskContextSupplier(),
config.populateMetaFields() ? Option.empty() :
- Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+ Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps())));
WriteStatus writeStatus = new WriteStatus(false, 0.0);
writeStatus.setStat(new HoodieWriteStat());
writeStatus.getStat().setNumWrites(0);
@@ -777,7 +516,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
.fromMetaClient(metaClient)
.setTimelineLayoutVersion(VERSION_0)
.setPopulateMetaFields(config.populateMetaFields())
- .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
+ .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
@@ -815,9 +554,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// verify that restore fails when "hoodie.archive.beyond.savepoint" is enabled.
SparkRDDWriteClient finalClient = client;
- assertThrows(IllegalArgumentException.class, () -> {
- finalClient.restoreToSavepoint("004");
- }, "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled");
+ assertThrows(IllegalArgumentException.class, () -> finalClient.restoreToSavepoint("004"),
+ "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled");
}
/**
@@ -826,9 +564,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testInsertsWithHoodieConcatHandle(boolean populateMetaFields) throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
- testHoodieConcatHandle(cfgBuilder.build(), false);
+ testHoodieConcatHandle(populateMetaFields, false);
}
/**
@@ -836,49 +572,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- testHoodieConcatHandle(cfgBuilder.build(), true);
- }
-
- /**
- * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
- *
- * @param config Write Config
- * @throws Exception in case of error
- */
- private void testHoodieConcatHandle(HoodieWriteConfig config, boolean isPrepped)
- throws Exception {
- // Force using older timeline layout
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
- .withProps(config.getProps()).withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(
- VERSION_0).build();
- HoodieTableMetaClient.withPropertyBuilder()
- .fromMetaClient(metaClient)
- .setTimelineLayoutVersion(VERSION_0)
- .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
- SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String newCommitTime = "001";
- String initCommitTime = "000";
- int numRecords = 200;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert,
- isPrepped, true, numRecords, config.populateMetaFields());
-
- // Write 2 (updates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- numRecords = 100;
- String commitTimeBetweenPrevAndNew = "002";
-
- final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
-
- writeBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime,
- numRecords, recordGenFunction, SparkRDDWriteClient::insert, true, numRecords, 300,
- 2, false, config.populateMetaFields());
+ testHoodieConcatHandle(true, true);
}
/**
@@ -886,8 +580,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testInsertsWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), false);
+ testHoodieConcatHandleOnDupInserts(false);
}
/**
@@ -895,37 +588,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testInsertsPreppedWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), true);
- }
-
- private void testHoodieConcatHandleOnDupInserts(HoodieWriteConfig config, boolean isPrepped) throws Exception {
- HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
- .withProps(config.getProps())
- .withMergeAllowDuplicateOnInserts(true)
- .build();
-
- SparkRDDWriteClient<RawTripTestPayload> client = getHoodieWriteClient(hoodieWriteConfig);
-
- // Write 1 (only inserts)
- String initCommitTime = "000";
- String newCommitTime = "001";
- int firstInsertRecords = 50;
- insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, firstInsertRecords, SparkRDDWriteClient::insert,
- isPrepped, true, firstInsertRecords, config.populateMetaFields());
-
- // Write 2 (updates with duplicates)
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
- List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
-
- final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUpdates);
-
- writeBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
- secondInsertRecords, recordGenFunction, SparkRDDWriteClient::insert, true, secondInsertRecords,
- firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+ testHoodieConcatHandleOnDupInserts(true);
}
@Test
@@ -949,12 +612,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
StoragePath completeRestoreFile = null;
StoragePath backupCompletedRestoreFile = null;
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
- final String commitTime1 = "001";
- client.startCommitWithTime(commitTime1);
- List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, 100);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
- List<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1).collect();
- assertNoWriteErrors(statuses);
+ insertBatchRecords(client, "001", 100, 3, 2, SparkRDDWriteClient::insert);
// inject a pending restore
client.savepoint("001", "user1", "comment1");
@@ -965,11 +623,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
metaClient.reloadActiveTimeline().getRestoreTimeline().filterCompletedInstants()
.getInstants().get(0);
completeRestoreFile = new StoragePath(
- config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
- + "/" + restoreCompleted.getFileName());
+ config.getBasePath() + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME
+ + StoragePath.SEPARATOR + restoreCompleted.getFileName());
backupCompletedRestoreFile = new StoragePath(
- config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
- + "/" + restoreCompleted.getFileName() + ".backup");
+ config.getBasePath() + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME
+ + StoragePath.SEPARATOR + restoreCompleted.getFileName() + ".backup");
metaClient.getStorage().rename(completeRestoreFile, backupCompletedRestoreFile);
}
@@ -982,14 +640,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
metaClient.getStorage().rename(backupCompletedRestoreFile, completeRestoreFile);
// retrigger a new commit, should succeed.
-
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
- final String commitTime3 = "003";
- client.startCommitWithTime(commitTime3);
- List<HoodieRecord> inserts3 = dataGen.generateInserts(commitTime3, 100);
- JavaRDD<HoodieRecord> insertRecordsRDD3 = jsc.parallelize(inserts3, 2);
- List<WriteStatus> statuses = client.insert(insertRecordsRDD3, commitTime3).collect();
- assertNoWriteErrors(statuses);
+ insertBatchRecords(client, "003", 100, 3, 2, SparkRDDWriteClient::insert);
}
}
@@ -998,47 +650,18 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testDeletes() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
- /**
- * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
- */
- String initCommitTime = "000";
- String newCommitTime = "001";
-
- final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
- Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
- List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
-
- recordsInFirstBatch.addAll(fewRecordsForInsert);
- recordsInFirstBatch.addAll(fewRecordsForDelete);
- return recordsInFirstBatch;
+ Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn =
+ (String instantTime, Integer numRecordsInThisCommit, List<HoodieRecord> recordsInFirstBatch) -> {
+ List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
+ return (time, numRecords) -> {
+ List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
+ List<HoodieRecord> fewRecordsForUpdate = recordsInFirstBatch.subList(50, 100);
+ recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+ recordsInSecondBatch.addAll(fewRecordsForUpdate);
+ return recordsInSecondBatch;
+ };
};
- writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
- // unused as genFn uses hard-coded number of inserts/updates/deletes
- -1, recordGenFunction, SparkRDDWriteClient::upsert, true, 200, 200, 1, false,
- true);
-
- /**
- * Write 2 (deletes+writes).
- */
- String prevCommitTime = newCommitTime;
- newCommitTime = "004";
- final List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
-
- recordGenFunction = (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
- List<HoodieRecord> fewRecordsForUpdate = recordsInFirstBatch.subList(50, 100);
- recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
- recordsInSecondBatch.addAll(fewRecordsForUpdate);
- return recordsInSecondBatch;
- };
- writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 100, recordGenFunction,
- SparkRDDWriteClient::upsert, true, 50, 150, 2, false,
- true);
+ super.testDeletes(secondBatchGenFn, 100, 50, 150);
}
/**
@@ -1049,85 +672,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testDeletesForInsertsInSameBatch() throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
- /**
- * Write 200 inserts and issue deletes to a subset(50) of inserts.
- */
- String initCommitTime = "000";
- String newCommitTime = "001";
-
- final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
- Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
- (String instantTime, Integer numRecordsInThisCommit) -> {
- List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
- List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
-
- recordsInFirstBatch.addAll(fewRecordsForInsert);
- recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
- return recordsInFirstBatch;
- };
-
- writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
- -1, recordGenFunction, SparkRDDWriteClient::upsert, true, 150, 150, 1, false,
- true);
- }
-
- private void assertPartitionPathRecordKeys(List<Pair<String, String>> expectedPartitionPathRecKeyPairs, String[] fullPartitionPaths) {
- Dataset<Row> rows = getAllRows(fullPartitionPaths);
- List<Pair<String, String>> actualPartitionPathRecKeyPairs = getActualPartitionPathAndRecordKeys(rows);
- // verify all partitionpath, record key matches
- assertActualAndExpectedPartitionPathRecordKeyMatches(expectedPartitionPathRecKeyPairs, actualPartitionPathRecKeyPairs);
- }
-
- private List<Pair<String, String>> getActualPartitionPathAndRecordKeys(Dataset<org.apache.spark.sql.Row> rows) {
- List<Pair<String, String>> actualPartitionPathRecKeyPairs = new ArrayList<>();
- for (Row row : rows.collectAsList()) {
- actualPartitionPathRecKeyPairs
- .add(Pair.of(row.getAs("_hoodie_partition_path"), row.getAs("_row_key")));
- }
- return actualPartitionPathRecKeyPairs;
- }
-
- private Dataset<org.apache.spark.sql.Row> getAllRows(String[] fullPartitionPaths) {
- return HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths);
- }
-
- private String getFullPartitionPath(String relativePartitionPath) {
- return getFullPartitionPaths(relativePartitionPath)[0];
- }
-
- private String[] getFullPartitionPaths(String... relativePartitionPaths) {
- String[] fullPartitionPaths = new String[relativePartitionPaths.length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, relativePartitionPaths[i]);
- }
- return fullPartitionPaths;
+ super.testDeletesForInsertsInSameBatch();
}
- private void assertActualAndExpectedPartitionPathRecordKeyMatches(List<Pair<String, String>> expectedPartitionPathRecKeyPairs,
- List<Pair<String, String>> actualPartitionPathRecKeyPairs) {
- // verify all partitionpath, record key matches
- assertEquals(expectedPartitionPathRecKeyPairs.size(), actualPartitionPathRecKeyPairs.size());
- for (Pair<String, String> entry : actualPartitionPathRecKeyPairs) {
- assertTrue(expectedPartitionPathRecKeyPairs.contains(entry));
- }
-
- for (Pair<String, String> entry : expectedPartitionPathRecKeyPairs) {
- assertTrue(actualPartitionPathRecKeyPairs.contains(entry));
- }
- }
-
- private Pair<List<WriteStatus>, List<HoodieRecord>> insertBatchRecords(SparkRDDWriteClient client, String commitTime,
- Integer recordNum, int expectStatueSize) {
+ private Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> insertBatchRecords(SparkRDDWriteClient client, String commitTime,
+ Integer recordNum, int expectStatusSize, int numSlices,
+ Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn) throws IOException {
client.startCommitWithTime(commitTime);
- List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime, recordNum);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
- List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime).collect();
- assertNoWriteErrors(statuses);
- assertEquals(expectStatueSize, statuses.size(), "check expect statue size.");
- return Pair.of(statuses, inserts1);
+ List<HoodieRecord> inserts = dataGen.generateInserts(commitTime, recordNum);
+ JavaRDD<HoodieRecord> insertRecordsRDD = jsc.parallelize(inserts, numSlices);
+ JavaRDD<WriteStatus> statuses = writeFn.apply(client, insertRecordsRDD, commitTime);
+ assertNoWriteErrors(statuses.collect());
+ assertEquals(expectStatusSize, statuses.count(), "check expect status size.");
+ return Pair.of(statuses, inserts);
}
@Test
@@ -1143,23 +700,20 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
HoodieSparkCopyOnWriteTable table = (HoodieSparkCopyOnWriteTable) HoodieSparkTable.create(config, context, metaClient);
//1. insert to generate 2 file group
- String commitTime1 = "001";
- Pair<List<WriteStatus>, List<HoodieRecord>> upsertResult = insertBatchRecords(client, commitTime1, 600, 2);
+ Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> upsertResult = insertBatchRecords(client, "001", 600, 2, 1, SparkRDDWriteClient::upsert);
List<HoodieRecord> inserts1 = upsertResult.getValue();
List<String> fileGroupIds1 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
.map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
assertEquals(2, fileGroupIds1.size());
// 2. generate clustering plan for fileGroupIds1 file groups
- String commitTime2 = "002";
List<List<FileSlice>> firstInsertFileSlicesList = table.getFileSystemView().getAllFileGroups(testPartitionPath)
.map(fileGroup -> fileGroup.getAllFileSlices().collect(Collectors.toList())).collect(Collectors.toList());
- List<FileSlice>[] fileSlices = (List<FileSlice>[]) firstInsertFileSlicesList.toArray(new List[firstInsertFileSlicesList.size()]);
- createRequestedReplaceInstant(this.metaClient, commitTime2, fileSlices);
+ List<FileSlice>[] fileSlices = (List<FileSlice>[]) firstInsertFileSlicesList.toArray(new List[0]);
+ createRequestedReplaceInstant(this.metaClient, "002", fileSlices);
// 3. insert one record with no updating reject exception, and not merge the small file, just generate a new file group
- String commitTime3 = "003";
- insertBatchRecords(client, commitTime3, 1, 1);
+ insertBatchRecords(client, "003", 1, 1, 1, SparkRDDWriteClient::upsert);
List<String> fileGroupIds2 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
.map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
assertEquals(3, fileGroupIds2.size());
@@ -1167,8 +721,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// 4. update one record for the clustering two file groups, throw reject update exception
String commitTime4 = "004";
client.startCommitWithTime(commitTime4);
- List<HoodieRecord> insertsAndUpdates3 = new ArrayList<>();
- insertsAndUpdates3.addAll(dataGen.generateUpdates(commitTime4, inserts1));
+ List<HoodieRecord> insertsAndUpdates3 = new ArrayList<>(dataGen.generateUpdates(commitTime4, inserts1));
String assertMsg = String.format("Not allowed to update the clustering files in partition: %s "
+ "For pending clustering operations, we are not going to support update for now.", testPartitionPath);
assertThrows(HoodieUpsertException.class, () -> {
@@ -1176,10 +729,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
}, assertMsg);
// 5. insert one record with no updating reject exception, will merge the small file
- String commitTime5 = "005";
- List<WriteStatus> statuses = insertBatchRecords(client, commitTime5, 1, 1).getKey();
+ JavaRDD<WriteStatus> statuses = insertBatchRecords(client, "005", 1, 1, 1, SparkRDDWriteClient::upsert).getKey();
fileGroupIds2.removeAll(fileGroupIds1);
- assertEquals(fileGroupIds2.get(0), statuses.get(0).getFileId());
+ assertEquals(fileGroupIds2.get(0), statuses.collect().get(0).getFileId());
List<String> firstInsertFileGroupIds4 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
.map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
assertEquals(3, firstInsertFileGroupIds4.size());
@@ -1258,7 +810,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(2, statuses.size(), "2 files needs to be committed.");
- HoodieTableMetaClient metadata = createMetaClient(basePath);
+ HoodieTableMetaClient metadata = createMetaClient();
HoodieTable table = getHoodieTable(metadata, config);
BaseFileOnlyView fileSystemView = table.getBaseFileOnlyView();
@@ -1308,7 +860,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
final String testPartitionPath = "2016/09/26";
final int insertSplitLimit = 100;
// setup the small file handling params
-
HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit, false, mergeAllowDuplicateInserts); // hold upto 200 records max
dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
SparkRDDWriteClient client = getHoodieWriteClient(config);
@@ -1316,14 +867,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// Inserts => will write file1
String commitTime1 = "001";
- client.startCommitWithTime(commitTime1);
- List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, insertSplitLimit); // this writes ~500kb
- Set<String> keys1 = recordsToRecordKeySet(inserts1);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
- List<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1).collect();
- assertNoWriteErrors(statuses);
+ Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> insertResult = insertBatchRecords(client, commitTime1,
+ insertSplitLimit, 1, 1, SparkRDDWriteClient::insert); // this writes ~500kb
+ List<WriteStatus> statuses = insertResult.getLeft().collect();
+ Set<String> keys1 = recordsToRecordKeySet(insertResult.getRight());
assertPartitionMetadata(basePath, new String[] {testPartitionPath}, storage);
- assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
@@ -1331,13 +879,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// Second, set of Inserts should just expand file1
String commitTime2 = "002";
- client.startCommitWithTime(commitTime2);
- List<HoodieRecord> inserts2 = dataGen.generateInserts(commitTime2, 40);
- Set<String> keys2 = recordsToRecordKeySet(inserts2);
- JavaRDD<HoodieRecord> insertRecordsRDD2 = jsc.parallelize(inserts2, 1);
- statuses = client.insert(insertRecordsRDD2, commitTime2).collect();
- assertNoWriteErrors(statuses);
- assertEquals(1, statuses.size(), "Just 1 file needs to be updated.");
+ insertResult = insertBatchRecords(client, commitTime2, 40, 1, 1, SparkRDDWriteClient::insert);
+ Set<String> keys2 = recordsToRecordKeySet(insertResult.getRight());
+ statuses = insertResult.getLeft().collect();
assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
@@ -1356,18 +900,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// Lots of inserts such that file1 is updated and expanded, a new file2 is created.
String commitTime3 = "003";
- client.startCommitWithTime(commitTime3);
- List<HoodieRecord> inserts3 = dataGen.generateInserts(commitTime3, 200);
- JavaRDD<HoodieRecord> insertRecordsRDD3 = jsc.parallelize(inserts3, 1);
- statuses = client.insert(insertRecordsRDD3, commitTime3).collect();
- assertNoWriteErrors(statuses);
- assertEquals(2, statuses.size(), "2 files needs to be committed.");
+ insertResult = insertBatchRecords(client, commitTime3, 200, 2, 1, SparkRDDWriteClient::insert);
+ statuses = insertResult.getLeft().collect();
assertEquals(340,
fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
"file should contain 340 records");
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
+ HoodieTableMetaClient metaClient = createMetaClient();
HoodieTable table = getHoodieTable(metaClient, config);
List<HoodieBaseFile> files = table.getBaseFileOnlyView()
.getLatestBaseFilesBeforeOrOn(testPartitionPath, commitTime3).collect(Collectors.toList());
@@ -1378,7 +918,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(commitTime3, file.getCommitTime(), "All files must be at commit 3");
totalInserts += fileUtils.readAvroRecords(storage, new StoragePath(file.getPath())).size();
}
- assertEquals(totalInserts, inserts1.size() + inserts2.size() + inserts3.size(), "Total number of records must add up");
+ assertEquals(340, totalInserts, "Total number of records must add up");
}
/**
@@ -1437,14 +977,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(0, statuses.size(), "Just 0 write status for delete.");
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(150,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + 150 + " records");
+ assertTheEntireDatasetHasAllRecordsStill(150);
// delete another batch. previous delete commit should have persisted the schema. If not,
// this will throw exception
@@ -1454,87 +987,32 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testSimpleClustering(boolean populateMetaFields) throws Exception {
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties())
- .build();
- testInsertAndClustering(clusteringConfig, populateMetaFields, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
+ testInsertAndClustering(createClusteringBuilder(true, 1).build(), populateMetaFields, true,
+ false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
}
@Test
public void testAndValidateClusteringOutputFiles() throws IOException {
- String partitionPath = "2015/03/16";
- testInsertTwoBatches(true, partitionPath);
-
- // Trigger clustering
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
- .withClusteringConfig(HoodieClusteringConfig.newBuilder().withInlineClustering(true).withInlineClusteringNumCommits(2)
- .fromProperties(getDisabledRowWriterProperties()).build());
- try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
- int numRecords = 200;
- String newCommitTime = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(newCommitTime, numRecords);
- client.startCommitWithTime(newCommitTime);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
- JavaRDD<WriteStatus> statuses = client.insert(insertRecordsRDD1, newCommitTime);
- client.commit(newCommitTime, statuses);
- List<WriteStatus> statusList = statuses.collect();
- assertNoWriteErrors(statusList);
-
- metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieInstant replaceCommitInstant =
- metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
- HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
- .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
- HoodieReplaceCommitMetadata.class);
-
- List<String> filesFromReplaceCommit = new ArrayList<>();
- replaceCommitMetadata.getPartitionToWriteStats()
- .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
-
- // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
- List<StoragePathInfo> pathInfoList =
- storage.listDirectEntries(new StoragePath(basePath + "/" + partitionPath));
- List<String> clusteredFiles = pathInfoList.stream()
- .filter(entry ->
- entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
- .map(pathInfo -> partitionPath + "/" + pathInfo.getPath().getName())
- .collect(Collectors.toList());
- assertEquals(clusteredFiles, filesFromReplaceCommit);
- }
+ testAndValidateClusteringOutputFiles(createBrokenClusteringClient, createClusteringBuilder(true, 2).build(), list2Rdd, rdd2List);
}
@Test
public void testRollbackOfRegularCommitWithPendingReplaceCommitInTimeline() throws Exception {
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties())
- .build();
// trigger clustering, but do not complete
- testInsertAndClustering(clusteringConfig, true, false, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
+ testInsertAndClustering(createClusteringBuilder(true, 1).build(), true, false,
+ false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
// trigger another partial commit, followed by valid commit. rollback of partial commit should succeed.
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
String commitTime1 = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
- client.startCommitWithTime(commitTime1);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
- JavaRDD<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1);
- List<WriteStatus> statusList = statuses.collect();
- assertNoWriteErrors(statusList);
+ insertBatchRecords(client, commitTime1, 200, 1, 2, SparkRDDWriteClient::upsert).getLeft();
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
+ HoodieTableMetaClient metaClient = createMetaClient();
assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterInflightsAndRequested().countInstants());
// trigger another commit. this should rollback latest partial commit.
- records1 = dataGen.generateInserts(commitTime1, 200);
- client.startCommitWithTime(commitTime1);
- insertRecordsRDD1 = jsc.parallelize(records1, 2);
- statuses = client.upsert(insertRecordsRDD1, commitTime1);
- statusList = statuses.collect();
- assertNoWriteErrors(statusList);
+ JavaRDD<WriteStatus> statuses = insertBatchRecords(client, commitTime1, 200, 1, 2, SparkRDDWriteClient::upsert).getLeft();
client.commit(commitTime1, statuses);
metaClient.reloadActiveTimeline();
// rollback should have succeeded. Essentially, the pending clustering should not hinder the rollback of regular commits.
@@ -1544,46 +1022,17 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testInlineScheduleClustering(boolean scheduleInlineClustering) throws IOException {
- testInsertTwoBatches(true);
-
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withAsyncClusteringMaxCommits(1).withInlineClustering(false).withScheduleInlineClustering(scheduleInlineClustering)
- .build();
-
- HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
- .withClusteringConfig(clusteringConfig)
- .withProps(getPropertiesForKeyGen()).build();
- SparkRDDWriteClient client = getHoodieWriteClient(config);
- dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
- String commitTime1 = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
- client.startCommitWithTime(commitTime1);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
- JavaRDD<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1);
- List<WriteStatus> statusList = statuses.collect();
- assertNoWriteErrors(statusList);
- client.commit(commitTime1, statuses);
-
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
- List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
- ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
- if (scheduleInlineClustering) {
- assertEquals(1, pendingClusteringPlans.size());
- } else {
- assertEquals(0, pendingClusteringPlans.size());
- }
+ HoodieClusteringConfig clusteringConfig = createClusteringBuilder(false, 1)
+ .withAsyncClusteringMaxCommits(1).withScheduleInlineClustering(scheduleInlineClustering).build();
+ testInlineScheduleClustering(createBrokenClusteringClient, clusteringConfig, list2Rdd, rdd2List);
}
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testClusteringWithSortColumns(boolean populateMetaFields) throws Exception {
// setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringSortColumns(populateMetaFields ? "_hoodie_record_key" : "_row_key")
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties())
- .build();
+ HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
+ .withClusteringSortColumns(populateMetaFields ? "_hoodie_record_key" : "_row_key").build();
testInsertAndClustering(clusteringConfig, populateMetaFields, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
}
@@ -1591,12 +1040,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@MethodSource("populateMetaFieldsParams")
public void testClusteringWithSortOneFilePerGroup(boolean populateMetaFields) throws Exception {
// setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
+ HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
.withClusteringSortColumns("begin_lat,begin_lon")
.withClusteringPlanStrategyClass(SparkSingleFileSortPlanStrategy.class.getName())
.withClusteringExecutionStrategyClass(SparkSingleFileSortExecutionStrategy.class.getName())
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
- .fromProperties(getDisabledRowWriterProperties())
.build();
// note that assertSameFileIds is true for this test because of the plan and execution strategy
testInsertAndClustering(clusteringConfig, populateMetaFields, true, true, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
@@ -1605,17 +1052,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testPendingClusteringRollback() throws Exception {
boolean populateMetaFields = true;
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties()).build();
-
// start clustering, but don't commit
- List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, populateMetaFields, false);
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
- List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
- ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
- assertEquals(1, pendingClusteringPlans.size());
+ List<HoodieRecord> allRecords = testInsertAndClustering(createClusteringBuilder(true, 1).build(), populateMetaFields, false);
+ HoodieTableMetaClient metaClient = createMetaClient();
+ List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans = getAndAssertPendingClusteringPlans(true, metaClient);
HoodieInstant pendingClusteringInstant = pendingClusteringPlans.get(0).getLeft();
// complete another commit after pending clustering
@@ -1635,7 +1075,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// delete rollback.completed instant to mimic failed rollback of clustering. and then trigger rollback of clustering again. same rollback instant should be used.
HoodieInstant rollbackInstant = metaClient.getActiveTimeline().getRollbackTimeline().lastInstant().get();
- FileCreateUtils.deleteRollbackCommit(metaClient.getBasePath(), rollbackInstant.getTimestamp());
+ FileCreateUtils.deleteRollbackCommit(metaClient.getBasePathV2().toString(), rollbackInstant.getTimestamp());
metaClient.reloadActiveTimeline();
// create replace commit requested meta file so that rollback will not throw FileNotFoundException
@@ -1645,7 +1085,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder()
.setClusteringPlan(clusteringPlan).setOperationType(WriteOperationType.CLUSTER.name()).build();
- FileCreateUtils.createRequestedReplaceCommit(metaClient.getBasePath(), pendingClusteringInstant.getTimestamp(), Option.of(requestedReplaceMetadata));
+ FileCreateUtils.createRequestedReplaceCommit(metaClient.getBasePathV2().toString(), pendingClusteringInstant.getTimestamp(), Option.of(requestedReplaceMetadata));
// trigger clustering again. no new rollback instants should be generated.
try {
@@ -1665,19 +1105,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ValueSource(booleans = {true, false})
public void testInflightClusteringRollbackWhenUpdatesAllowed(boolean rollbackPendingClustering) throws Exception {
// setup clustering config with update strategy to allow updates during ingestion
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder()
- .withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0)
+ HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
.withClusteringUpdatesStrategy("org.apache.hudi.client.clustering.update.strategy.SparkAllowUpdateStrategy")
- .withRollbackPendingClustering(rollbackPendingClustering)
- .withInlineClustering(true).withInlineClusteringNumCommits(1)
- .fromProperties(getDisabledRowWriterProperties()).build();
+ .withRollbackPendingClustering(rollbackPendingClustering).build();
// start clustering, but don't commit keep it inflight
List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, true, false);
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
- List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
- ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
- assertEquals(1, pendingClusteringPlans.size());
+ HoodieTableMetaClient metaClient = createMetaClient();
+ List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans = getAndAssertPendingClusteringPlans(true, metaClient);
HoodieInstant pendingClusteringInstant = pendingClusteringPlans.get(0).getLeft();
assertEquals(pendingClusteringInstant.getState(), INFLIGHT);
@@ -1685,6 +1120,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(EAGER);
addConfigsForPopulateMetaFields(cfgBuilder, true);
cfgBuilder.withClusteringConfig(clusteringConfig);
+ cfgBuilder.withProperties(getPropertiesForKeyGen());
HoodieWriteConfig config = cfgBuilder.build();
SparkRDDWriteClient client = getHoodieWriteClient(config);
String commitTime = client.createNewInstantTime();
@@ -1699,11 +1135,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testClusteringWithFailingValidator() throws Exception {
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringSortColumns("_hoodie_record_key").withInlineClustering(true)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
- .fromProperties(getDisabledRowWriterProperties()).build();
+ HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
+ .withClusteringSortColumns("_hoodie_record_key").withInlineClustering(true).build();
try {
testInsertAndClustering(clusteringConfig, true, true, false, FailingPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
fail("expected pre-commit clustering validation to fail");
@@ -1714,12 +1147,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testClusteringInvalidConfigForSqlQueryValidator() throws Exception {
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties()).build();
try {
- testInsertAndClustering(clusteringConfig, false, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), "", "");
+ testInsertAndClustering(createClusteringBuilder(true, 1).build(), false, true,
+ false, SqlQueryEqualityPreCommitValidator.class.getName(), "", "");
fail("expected pre-commit clustering validation to fail because sql query is not configured");
} catch (HoodieValidationException e) {
// expected
@@ -1728,24 +1158,16 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testClusteringInvalidConfigForSqlQuerySingleResultValidator() throws Exception {
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties()).build();
-
- testInsertAndClustering(clusteringConfig, false, true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
+ testInsertAndClustering(createClusteringBuilder(true, 1).build(), false, true,
+ false, SqlQuerySingleResultPreCommitValidator.class.getName(),
"", COUNT_SQL_QUERY_FOR_VALIDATION + "#400");
}
@Test
public void testClusteringInvalidConfigForSqlQuerySingleResultValidatorFailure() throws Exception {
- // setup clustering config.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
- .fromProperties(getDisabledRowWriterProperties()).build();
-
try {
- testInsertAndClustering(clusteringConfig, false, true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
+ testInsertAndClustering(createClusteringBuilder(true, 1).build(), false,
+ true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
"", COUNT_SQL_QUERY_FOR_VALIDATION + "#802");
fail("expected pre-commit clustering validation to fail because of count mismatch. expect 400 rows, not 802");
} catch (HoodieValidationException e) {
@@ -1760,136 +1182,16 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private List<HoodieRecord> testInsertAndClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields,
boolean completeClustering, boolean assertSameFileIds, String validatorClasses,
String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation) throws Exception {
- Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields);
- testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords);
+ Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields, createBrokenClusteringClient);
+ testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation,
+ sqlQueryForSingleResultValidation, allRecords, clusteringMetadataRdd2List, createKeyGenerator);
return allRecords.getLeft().getLeft();
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail) throws IOException {
- try {
- Properties properties = new Properties();
- properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
- properties.setProperty("hoodie.auto.commit", "false");
- properties.setProperty("hoodie.clustering.inline.max.commits", "1");
- properties.setProperty("hoodie.clustering.inline", "true");
- testInsertTwoBatches(true, "2015/03/16", properties, true);
- assertFalse(shouldFail);
- } catch (HoodieException e) {
- assertEquals(CLUSTERING_FAILURE, e.getMessage());
- assertTrue(shouldFail);
- }
- }
-
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields) throws IOException {
- return testInsertTwoBatches(populateMetaFields, "2015/03/16");
- }
-
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath) throws IOException {
- return testInsertTwoBatches(populateMetaFields, partitionPath, new Properties(), false);
- }
-
- /**
- * This method returns following three items:
- * 1. List of all HoodieRecord written in the two batches of insert.
- * 2. Commit instants of the two batches.
- * 3. List of new file group ids that were written in the two batches.
- */
- private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props,
- boolean failInlineClustering) throws IOException {
- // create config to not update small files.
- HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
- populateMetaFields ? props : getPropertiesForKeyGen());
- SparkRDDWriteClient client;
- if (failInlineClustering) {
- if (null != writeClient) {
- writeClient.close();
- writeClient = null;
- }
- client = new WriteClientBrokenClustering(context, config);
- } else {
- client = getHoodieWriteClient(config);
- }
-
- dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
- String commitTime1 = client.createNewInstantTime();
- List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
- List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
- Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
-
- String commitTime2 = client.createNewInstantTime();
- List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
- List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
- client.close();
- Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
- Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
- fileIdsUnion.addAll(fileIds2);
- //verify new files are created for 2nd write
- Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
- fileIdIntersection.retainAll(fileIds2);
- assertEquals(0, fileIdIntersection.size());
- return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
- }
-
- private void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
- String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
- Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords) throws IOException {
-
- HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
- .withClusteringConfig(clusteringConfig)
- .withProps(getPropertiesForKeyGen()).build();
- HoodieWriteMetadata<JavaRDD<WriteStatus>> clusterMetadata =
- performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords.getLeft());
- if (assertSameFileIds) {
- Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
- .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
- Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
- assertEquals(insertedFileIds, replacedFileIds);
- }
- if (completeClustering) {
- String clusteringCommitTime = metaClient.reloadActiveTimeline().getCompletedReplaceTimeline()
- .getReverseOrderedInstants().findFirst().get().getTimestamp();
- verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(), clusterMetadata.getWriteStatuses().collect(), config);
- }
- }
-
- private HoodieWriteMetadata<JavaRDD<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
- boolean populateMetaFields,
- boolean completeClustering,
- String validatorClasses,
- String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
- Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException {
- HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
- .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
- .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
- .build();
-
- HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
- .withPreCommitValidatorConfig(validatorConfig)
- .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
- .withClusteringConfig(clusteringConfig).build();
-
- // create client with new config.
- SparkRDDWriteClient client = getHoodieWriteClient(config);
- String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
- HoodieWriteMetadata<JavaRDD<WriteStatus>> clusterMetadata = client.cluster(clusteringCommitTime, completeClustering);
- if (config.populateMetaFields()) {
- verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses().collect());
- } else {
- verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses().collect(), config);
- }
-
- Set<HoodieFileGroupId> replacedFileIds = new HashSet<>();
- clusterMetadata.getPartitionToReplaceFileIds().entrySet().forEach(partitionFiles ->
- partitionFiles.getValue().stream().forEach(file ->
- replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file))));
- return clusterMetadata;
- }
-
- private Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
- return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
+ testFailWritesOnInlineTableServiceExceptions(shouldFail, createBrokenClusteringClient);
}
/**
@@ -1926,8 +1228,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private void verifyInsertOverwritePartitionHandling(int batch1RecordsCount, int batch2RecordsCount, boolean populateMetaFields) throws Exception {
final String testPartitionPath = "americas";
HoodieWriteConfig config = getSmallInsertWriteConfig(2000,
- TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
- ? new Properties() : getPropertiesForKeyGen());
+ TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, getPropertiesForKeyGen(populateMetaFields));
SparkRDDWriteClient client = getHoodieWriteClient(config);
dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
@@ -1940,19 +1241,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
String commitTime2 = "002";
client.startCommitWithTime(commitTime2, REPLACE_COMMIT_ACTION);
List<HoodieRecord> inserts2 = dataGen.generateInserts(commitTime2, batch2RecordsCount);
- List<HoodieRecord> insertsAndUpdates2 = new ArrayList<>();
- insertsAndUpdates2.addAll(inserts2);
+ List<HoodieRecord> insertsAndUpdates2 = new ArrayList<>(inserts2);
JavaRDD<HoodieRecord> insertAndUpdatesRDD2 = jsc.parallelize(insertsAndUpdates2, 2);
HoodieWriteResult writeResult = client.insertOverwrite(insertAndUpdatesRDD2, commitTime2);
statuses = writeResult.getWriteStatuses().collect();
assertNoWriteErrors(statuses);
assertEquals(batch1Buckets, new HashSet<>(writeResult.getPartitionToReplaceFileIds().get(testPartitionPath)));
- verifyRecordsWritten(commitTime2, populateMetaFields, inserts2, statuses, config);
+ verifyRecordsWritten(commitTime2, populateMetaFields, inserts2, statuses, config,
+ HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps()));
}
private Set<String> getFileIdsFromWriteStatus(List<WriteStatus> statuses) {
- return statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
+ return statuses.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
}
/**
@@ -1986,8 +1287,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1).collect();
assertNoWriteErrors(statuses);
- Set<String> batchBuckets = statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
- verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig());
+ Set<String> batchBuckets = statuses.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
+ verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig(),
+ HoodieSparkKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
return batchBuckets;
}
@@ -2010,8 +1312,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private void verifyDeletePartitionsHandling(int batch1RecordsCount, int batch2RecordsCount, int batch3RecordsCount,
boolean populateMetaFields) throws Exception {
HoodieWriteConfig config = getSmallInsertWriteConfig(2000,
- TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
- ? new Properties() : getPropertiesForKeyGen());
+ TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, getPropertiesForKeyGen(populateMetaFields));
SparkRDDWriteClient client = getHoodieWriteClient(config);
dataGen = new HoodieTestDataGenerator();
@@ -2061,78 +1362,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(0, baseFiles.size());
}
- /**
- * Verify data in base files matches expected records and commit time.
- */
- private void verifyRecordsWritten(String commitTime, boolean populateMetadataField,
- List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, HoodieWriteConfig config) throws IOException {
- List<GenericRecord> records = new ArrayList<>();
- Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
- if (config.populateMetaFields()) {
- for (GenericRecord record : records) {
- String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- assertEquals(commitTime,
- record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
- assertTrue(expectedKeys.contains(recordKey));
- }
- } else {
- KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()));
- for (GenericRecord record : records) {
- String recordKey = keyGenerator.getKey(record).getRecordKey();
- if (!populateMetadataField) {
- assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
- }
- assertTrue(expectedKeys.contains(recordKey));
- }
- }
- }
-
- @NotNull
- private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
- for (WriteStatus status : allStatus) {
- StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
- }
- Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
- assertEquals(records.size(), expectedKeys.size());
- return expectedKeys;
- }
-
- private void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
- List<GenericRecord> records = new ArrayList<>();
- Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
- Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
- .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
- assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
- Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
- for (GenericRecord record : records) {
- String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- assertTrue(expectedKeys.contains(recordKey));
- String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
- assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
- }
- }
-
- private List<WriteStatus> writeAndVerifyBatch(SparkRDDWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
- return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
- }
-
- private List<WriteStatus> writeAndVerifyBatch(SparkRDDWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
- client.startCommitWithTime(commitTime);
- JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts, 2);
- JavaRDD<WriteStatus> statusRDD = client.upsert(insertRecordsRDD1, commitTime);
- if (autoCommitOff) {
- client.commit(commitTime, statusRDD);
- }
- List<WriteStatus> statuses = statusRDD.collect();
- assertNoWriteErrors(statuses);
- verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig());
-
- return statuses;
- }
-
private Pair<Set<String>, List<HoodieRecord>> testUpdates(String instantTime, SparkRDDWriteClient client,
- int sizeToInsertAndUpdate, int expectedTotalRecords)
+ int sizeToInsertAndUpdate, int expectedRecords)
throws IOException {
client.startCommitWithTime(instantTime);
List<HoodieRecord> inserts = dataGen.generateInserts(instantTime, sizeToInsertAndUpdate);
@@ -2145,14 +1376,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
List<WriteStatus> statuses = client.upsert(insertAndUpdatesRDD, instantTime).collect();
assertNoWriteErrors(statuses);
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(expectedTotalRecords,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + expectedTotalRecords + " records");
+ assertTheEntireDatasetHasAllRecordsStill(expectedRecords);
return Pair.of(keys, inserts);
}
@@ -2169,14 +1393,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
assertEquals(existingFile, statuses.get(0).getFileId(), "Existing file should be expanded");
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(expectedRecords,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + expectedRecords + " records");
+ assertTheEntireDatasetHasAllRecordsStill(expectedRecords);
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
assertEquals(expectedRecords,
@@ -2191,29 +1408,24 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
}
}
+ @Override
+ protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+ String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
+ for (int i = 0; i < fullPartitionPaths.length; i++) {
+ fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
+ }
+ assertEquals(expectedRecords,
+ HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(), "Must contain " + expectedRecords + " records");
+ return fullPartitionPaths;
+ }
+
/**
* Test delete with delete api.
*/
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testDeletesWithoutInserts(boolean populateMetaFields) {
- final String testPartitionPath = "2016/09/26";
- final int insertSplitLimit = 100;
- // setup the small file handling params
- HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
- TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
- ? new Properties() : getPropertiesForKeyGen());
- dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
- SparkRDDWriteClient client = getHoodieWriteClient(config);
-
- // delete non existent keys
- String commitTime1 = "001";
- client.startCommitWithTime(commitTime1);
-
- List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
- List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
- JavaRDD<HoodieKey> deleteKeys = jsc.parallelize(hoodieKeysToDelete, 1);
- client.delete(deleteKeys, commitTime1).collect();
+ testDeletesWithoutInserts(populateMetaFields, list2Rdd, rdd2List);
}
/**
@@ -2221,47 +1433,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testCommitWritesRelativePaths() throws Exception {
-
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, true);
- try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
- HoodieSparkTable table = HoodieSparkTable.create(cfgBuilder.build(), context, metaClient);
-
- String instantTime = "000";
- client.startCommitWithTime(instantTime);
-
- List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
- JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
-
- JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-
- assertTrue(client.commit(instantTime, result), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
-
- // Get base file paths from commit metadata
- String actionType = metaClient.getCommitActionType();
- HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
- HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(),
- HoodieCommitMetadata.class);
- String basePath = table.getMetaClient().getBasePath();
- Collection<String> commitPathNames =
- commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
-
- // Read from commit file
- HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline()
- .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
- HoodieCommitMetadata.class);
- HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
- // Compare values in both to make sure they are equal.
- for (String pathName : paths.values()) {
- assertTrue(commitPathNames.contains(pathName));
- }
- }
+ testCommitWritesRelativePaths(list2Rdd);
}
/**
@@ -2270,60 +1442,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@MethodSource("populateMetaFieldsParams")
public void testMetadataStatsOnCommit(boolean populateMetaFields) throws Exception {
- HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
- HoodieWriteConfig cfg = cfgBuilder.build();
- SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-
- String instantTime0 = "000";
- client.startCommitWithTime(instantTime0);
-
- List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
- JavaRDD<HoodieRecord> writeRecords0 = jsc.parallelize(records0, 1);
- JavaRDD<WriteStatus> result0 = client.bulkInsert(writeRecords0, instantTime0);
-
- assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime0),
- "After explicit commit, commit file should be created");
-
- // Read from commit file
- HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
- HoodieCommitMetadata.class);
- int inserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
- }
- }
- assertEquals(200, inserts);
-
- // Update + Inserts such that they just expand file1
- String instantTime1 = "001";
- client.startCommitWithTime(instantTime1);
-
- List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
- JavaRDD<HoodieRecord> writeRecords1 = jsc.parallelize(records1, 1);
- JavaRDD<WriteStatus> result1 = client.upsert(writeRecords1, instantTime1);
-
- assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime1),
- "After explicit commit, commit file should be created");
-
- // Read from commit file
- metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
- HoodieCommitMetadata.class);
- inserts = 0;
- int upserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
- upserts += stat.getNumUpdateWrites();
- }
- }
- assertEquals(0, inserts);
- assertEquals(200, upserts);
+ testMetadataStatsOnCommit(populateMetaFields, list2Rdd);
}
/**
@@ -2332,74 +1451,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
- String instantTime = "000";
- HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
- SparkRDDWriteClient client = getHoodieWriteClient(cfg);
- Pair<StoragePath, JavaRDD<WriteStatus>> result = testConsistencyCheck(
- metaClient, instantTime, enableOptimisticConsistencyGuard);
-
- // Delete orphan marker and commit should succeed
- metaClient.getStorage().deleteFile(result.getKey());
- if (!enableOptimisticConsistencyGuard) {
- assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
- // Marker directory must be removed
- assertFalse(metaClient.getStorage()
- .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- } else {
- // with optimistic, first client.commit should have succeeded.
- assertTrue(testTable.commitExists(instantTime),
- "After explicit commit, commit file should be created");
- // Marker directory must be removed
- assertFalse(metaClient.getStorage()
- .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- }
+ testConsistencyCheckDuringFinalize(context, enableOptimisticConsistencyGuard, getHoodieTable, list2Rdd, rdd2List);
}
private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
boolean populateMetaFields) throws Exception {
- String instantTime = "00000000000010";
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
- Properties properties = new Properties();
- if (!populateMetaFields) {
- properties = getPropertiesForKeyGen();
- }
-
- HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build() :
- getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
- .withConsistencyCheckEnabled(true)
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
- .withOptimisticConsistencyGuardSleepTimeMs(1).build())
- .withProperties(properties).build();
- SparkRDDWriteClient client = getHoodieWriteClient(cfg);
- testConsistencyCheck(metaClient, instantTime, enableOptimisticConsistencyGuard);
-
- if (!enableOptimisticConsistencyGuard) {
- // Rollback of this commit should succeed with FailSafeCG
- client.rollback(instantTime);
- assertFalse(testTable.commitExists(instantTime),
- "After explicit rollback, commit file should not be present");
- // Marker directory must be removed after rollback
- assertFalse(metaClient.getStorage().exists(
- new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- } else {
- // if optimistic CG is enabled, commit should have succeeded.
- assertTrue(testTable.commitExists(instantTime),
- "With optimistic CG, first commit should succeed. commit file should be present");
- // Marker directory must be removed after rollback
- assertFalse(metaClient.getStorage().exists(
- new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
- client.rollback(instantTime);
- assertFalse(testTable.commitExists(instantTime),
- "After explicit rollback, commit file should not be present");
- }
+ testRollbackAfterConsistencyCheckFailureUsingFileList(context, rollbackUsingMarkers, enableOptimisticConsistencyGuard,
+ populateMetaFields, getHoodieTable, list2Rdd, rdd2List);
}
@ParameterizedTest
@@ -2414,257 +1472,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
testRollbackAfterConsistencyCheckFailureUsingFileList(true, enableOptimisticConsistencyGuard, populateMetCols);
}
- //@ParameterizedTest
- //@MethodSource("rollbackFailedCommitsParams")
@Test
public void testRollbackFailedCommits() throws Exception {
- // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
- HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
- boolean populateMetaFields = true;
- HoodieTestUtils.init(storageConf, basePath);
- SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-
- // perform 1 successful commit
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, true);
-
- // Perform 2 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // refresh data generator to delete records generated from failed commits
- dataGen = new HoodieTestDataGenerator();
- // Perform 1 successful write
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, true);
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
- assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
- assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
- assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
- // Await till enough time passes such that the first 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
- Thread.sleep(2000);
- }
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- // Perform 1 successful write
- writeBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, true);
- client.clean();
- client.close();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- if (cleaningPolicy.isLazy()) {
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
- .countInstants()
- == 2);
- // Since we write rollbacks not clean, there should be no clean action on the timeline
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
- .countInstants()
- == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- } else if (cleaningPolicy.isNever()) {
- // never will get translated to Lazy if OCC is enabled.
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
- .countInstants()
- == 2);
- // There should be no clean or rollback action on the timeline
- assertTrue(
- timeline
- .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
- .countInstants()
- == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- }
+ super.testRollbackFailedCommits(true);
}
@Test
public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception {
- boolean populateMetaFields = true;
- HoodieTestUtils.init(storageConf, basePath);
- HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
- SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- // Perform 1 successful writes to table
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, true);
-
- // Perform 1 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Toggle cleaning policy to LAZY
- cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
- // Perform 2 failed writes to table
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Await till enough time passes such that the 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
- Thread.sleep(2000);
- }
- client.clean();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
- // Perform 2 failed commits
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- writeBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
- 0, false);
- client.close();
- // Toggle cleaning policy to EAGER
- cleaningPolicy = EAGER;
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- client.startCommit();
- client.close();
- timeline = metaClient.getActiveTimeline().reload();
- // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1);
+ super.testRollbackFailedCommitsToggleCleaningPolicy(true);
}
@Test
public void testParallelInsertAndCleanPreviousFailedCommits() throws Exception {
- HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
- ExecutorService service = Executors.newFixedThreadPool(2);
- HoodieTestUtils.init(storageConf, basePath);
- SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // perform 1 successful write
- writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
- 0, true);
-
- // Perform 2 failed writes to table
- writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
- 0, false);
- client.close();
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
- 100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
- 0, false);
- client.close();
- // refresh data generator to delete records generated from failed commits
- dataGen = new HoodieTestDataGenerator();
- // Create a successful commit
- Future<JavaRDD<WriteStatus>> commit3 = service.submit(() -> writeBatch(new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
- "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
- SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
- commit3.get();
- HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
- assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
- assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
- assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
- client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // Await till enough time passes such that the first 2 failed commits heartbeats are expired
- boolean conditionMet = false;
- while (!conditionMet) {
- conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
- Thread.sleep(2000);
- }
- Future<JavaRDD<WriteStatus>> commit4 = service.submit(() -> writeBatch(new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
- "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
- SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
- Future<HoodieCleanMetadata> clean1 = service.submit(() -> new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)).clean());
- commit4.get();
- clean1.get();
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 2);
- // Since we write rollbacks not clean, there should be no clean action on the timeline
- assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(CLEAN_ACTION)).countInstants() == 0);
- assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
- client.close();
- }
-
- private Pair<StoragePath, JavaRDD<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
- throws Exception {
- HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
- .build()) : (getConfigBuilder().withAutoCommit(false)
- .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
- .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
- .withOptimisticConsistencyGuardSleepTimeMs(1).build())
- .build());
- SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-
- client.startCommitWithTime(instantTime);
- JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(instantTime, 200), 1);
- JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
- result.collect();
-
- // Create a dummy marker file to simulate the case that a marker file was created without data file.
- // This should fail the commit
- String partitionPath;
- String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
- if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
- String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
- markerFolderPath, storage, context, 1).values().stream()
- .flatMap(Collection::stream).findFirst().get();
- partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
- } else {
- partitionPath = storage.globEntries(
- new StoragePath(String.format("%s/*/*/*/*", markerFolderPath)), path ->
- path.toString().contains(HoodieTableMetaClient.MARKER_EXTN))
- .stream()
- .limit(1).map(status -> status.getPath().getParent().toString())
- .collect(Collectors.toList()).get(0);
- }
-
- Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
- cfg.getMarkersType(), getHoodieTable(metaClient, cfg), instantTime)
- .create(partitionPath,
- FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(), BASE_FILE_EXTENSION),
- IOType.MERGE);
- if (!enableOptimisticConsistencyGuard) {
- Exception e = assertThrows(HoodieCommitException.class, () -> {
- client.commit(instantTime, result);
- }, "Commit should fail due to consistency check");
- assertTrue(e.getCause() instanceof HoodieIOException);
- } else {
- // with optimistic CG, commit should succeed
- client.commit(instantTime, result);
- }
- return Pair.of(markerFilePath.get(), result);
+ super.testParallelInsertAndCleanPreviousFailedCommits(true);
}
@Test
@@ -2680,17 +1500,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1);
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, firstInstantTime);
assertTrue(client.commit(firstInstantTime, result), "Commit should succeed");
- assertTrue(testTable.commitExists(firstInstantTime),
- "After explicit commit, commit file should be created");
+ assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created");
- // Check the entire dataset has all records still
- String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
- for (int i = 0; i < fullPartitionPaths.length; i++) {
- fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
- }
- assertEquals(numRecords,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + numRecords + " records");
+ String[] fullPartitionPaths = assertTheEntireDatasetHasAllRecordsStill(numRecords);
String nextInstantTime = "0001";
client.startCommitWithTime(nextInstantTime);
@@ -2699,24 +1511,18 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<WriteStatus> inserts = client.bulkInsert(insertRecords, nextInstantTime);
JavaRDD<WriteStatus> upserts = client.upsert(updateRecords, nextInstantTime);
assertTrue(client.commit(nextInstantTime, inserts.union(upserts)), "Commit should succeed");
- assertTrue(testTable.commitExists(firstInstantTime),
- "After explicit commit, commit file should be created");
+ assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created");
int totalRecords = 2 * numRecords;
assertEquals(totalRecords,
- HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
- "Must contain " + totalRecords + " records");
+ HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(), "Must contain " + totalRecords + " records");
}
@Test
public void testClusteringCommitInPresenceOfInflightCommit() throws Exception {
Properties properties = getDisabledRowWriterProperties();
properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + "/.hoodie/.locks");
- HoodieLockConfig lockConfig = HoodieLockConfig.newBuilder()
- .withLockProvider(FileSystemBasedLockProviderTestClass.class)
- .withConflictResolutionStrategy(new PreferWriterConflictResolutionStrategy())
- .build();
- HoodieCleanConfig cleanConfig = HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).withAutoClean(false).build();
+ HoodieLockConfig lockConfig = createLockConfig(new PreferWriterConflictResolutionStrategy());
+ HoodieCleanConfig cleanConfig = createCleanConfig(HoodieFailedWritesCleaningPolicy.LAZY, false);
HoodieWriteConfig insertWriteConfig = getConfigBuilder().withAutoCommit(false)
.withCleanConfig(cleanConfig)
.withLockConfig(lockConfig)
@@ -2741,17 +1547,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
2, false);
// Schedule and execute a clustering plan on the same partition. During conflict resolution the commit should fail.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
- .build();
- HoodiePreCommitValidatorConfig preCommitValidatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(StringUtils.nullToEmpty(SqlQuerySingleResultPreCommitValidator.class.getName()))
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#200")
- .build();
HoodieWriteConfig clusteringWriteConfig = getConfigBuilder().withAutoCommit(false)
.withCleanConfig(cleanConfig)
- .withClusteringConfig(clusteringConfig)
- .withPreCommitValidatorConfig(preCommitValidatorConfig)
+ .withClusteringConfig(createClusteringBuilder(true, 1).build())
+ .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200))
.withLockConfig(lockConfig)
.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
.withProperties(properties)
@@ -2779,15 +1578,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
public void testIngestionCommitInPresenceOfCompletedClusteringCommit() throws Exception {
Properties properties = getDisabledRowWriterProperties();
properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + "/.hoodie/.locks");
- HoodieLockConfig lockConfig = HoodieLockConfig.newBuilder()
- .withLockProvider(FileSystemBasedLockProviderTestClass.class)
- .withConflictResolutionStrategy(new PreferWriterConflictResolutionStrategy())
- .build();
- HoodieCleanConfig cleanConfig = HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).withAutoClean(false).build();
+ HoodieCleanConfig cleanConfig = createCleanConfig(HoodieFailedWritesCleaningPolicy.LAZY, false);
HoodieWriteConfig insertWriteConfig = getConfigBuilder().withAutoCommit(false)
.withCleanConfig(cleanConfig)
- .withLockConfig(lockConfig)
+ .withLockConfig(createLockConfig(new PreferWriterConflictResolutionStrategy()))
.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
.withProperties(properties)
.build();
@@ -2809,27 +1603,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
2, false);
// Schedule and execute a clustering plan on the same partition. During conflict resolution the commit should fail.
- HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
- .withClusteringTargetPartitions(0)
- .withInlineClusteringNumCommits(1)
- .build();
-
// Since it is harder to test corner cases where the ingestion writer is at dedupe step right before the inflight file creation
// and clustering commit is just about to complete, using the default conflict resolutions strategy only for clustering job
// To create a successful commit.
- HoodieLockConfig clusteringLockConfig = HoodieLockConfig.newBuilder()
- .withLockProvider(FileSystemBasedLockProviderTestClass.class)
- .withConflictResolutionStrategy(new SimpleConcurrentFileWritesConflictResolutionStrategy())
- .build();
- HoodiePreCommitValidatorConfig preCommitValidatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
- .withPreCommitValidator(StringUtils.nullToEmpty(SqlQuerySingleResultPreCommitValidator.class.getName()))
- .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#200")
- .build();
HoodieWriteConfig clusteringWriteConfig = getConfigBuilder().withAutoCommit(false)
.withCleanConfig(cleanConfig)
- .withClusteringConfig(clusteringConfig)
- .withPreCommitValidatorConfig(preCommitValidatorConfig)
- .withLockConfig(clusteringLockConfig)
+ .withClusteringConfig(createClusteringBuilder(true, 1).build())
+ .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200))
+ .withLockConfig(createLockConfig(new SimpleConcurrentFileWritesConflictResolutionStrategy()))
.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
.withProperties(properties)
.build();
@@ -2847,75 +1628,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertThrows(HoodieWriteConflictException.class, () -> client.commit(inflightCommit, ingestionResult));
}
- /**
- * Build Hoodie Write Config for small data file sizes.
- */
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize) {
- return getSmallInsertWriteConfig(insertSplitSize, false);
- }
-
- /**
- * Build Hoodie Write Config for small data file sizes.
- */
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema) {
- return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, false);
- }
-
- /**
- * Build Hoodie Write Config for small data file sizes.
- */
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, boolean mergeAllowDuplicateInserts) {
- return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, dataGen.getEstimatedFileSizeInBytes(150), mergeAllowDuplicateInserts);
- }
-
- /**
- * Build Hoodie Write Config for specified small file sizes.
- */
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize) {
- return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, smallFileSize, false);
- }
-
- /**
- * Build Hoodie Write Config for specified small file sizes.
- */
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize, boolean mergeAllowDuplicateInserts) {
- String schemaStr = useNullSchema ? NULL_SCHEMA : TRIP_EXAMPLE_SCHEMA;
- return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts);
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize) {
- return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false);
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
- return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts, true, new Properties());
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
- return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
- }
-
- private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
- boolean populateMetaFields, Properties props) {
- HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
- if (!populateMetaFields) {
- builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.SIMPLE).build());
- }
- return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
- .compactionSmallFileSize(smallFileSize)
- // Set rollback to LAZY so no inflights are deleted
- .insertSplitSize(insertSplitSize).build())
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
- .build())
- .withStorageConfig(HoodieStorageConfig.newBuilder()
- .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
- .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
- .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
- .withProps(props)
- .build();
- }
-
protected HoodieInstant createRequestedReplaceInstant(HoodieTableMetaClient metaClient, String clusterTime, List<FileSlice>[] fileSlices) throws IOException {
HoodieClusteringPlan clusteringPlan =
ClusteringUtils.createClusteringPlan(EXECUTION_STRATEGY_CLASS_NAME.defaultValue(), STRATEGY_PARAMS, fileSlices, Collections.emptyMap());
@@ -2927,32 +1639,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
return clusteringInstant;
}
- private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
- Properties properties = new Properties();
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
- properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
- if (!populateMetaFields) {
- getPropertiesForKeyGen(populateMetaFields).entrySet().forEach(kv ->
- properties.put(kv.getKey(), kv.getValue()));
- }
- return getConfigBuilder()
- .withEmbeddedTimelineServerEnabled(false)
- .withCleanConfig(HoodieCleanConfig.newBuilder()
- .withFailedWritesCleaningPolicy(cleaningPolicy)
- .withAutoClean(false).build())
- .withTimelineLayoutVersion(1)
- .withHeartbeatIntervalInMs(3 * 1000)
- .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
- .withRemoteServerPort(timelineServicePort).build())
- .withAutoCommit(false)
- .withLockConfig(HoodieLockConfig.newBuilder()
- .withLockProvider(InProcessLockProvider.class)
- .build())
- .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
- .withProperties(properties).build();
- }
-
public static class FailingPreCommitValidator<T extends HoodieRecordPayload, I, K, O extends HoodieData<WriteStatus>> extends SparkPreCommitValidator<T, I, K, O> {
public FailingPreCommitValidator(HoodieSparkTable table, HoodieEngineContext context, HoodieWriteConfig config) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
index 2fceb1057da..107ce6a84de 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
@@ -52,6 +52,8 @@ import java.util.List;
import java.util.function.Function;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForKeys;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForRecords;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -177,6 +179,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
* @param wrapped Actual Records Generation function
* @return Wrapped Function
*/
+ @Override
public Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI,
HoodieWriteConfig writeConfig,
Function2<List<HoodieRecord>, String, Integer> wrapped) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index 5b1f514982b..eefa825bc5c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -21,9 +21,11 @@ import org.apache.hudi.HoodieConversionUtils;
import org.apache.hudi.avro.model.HoodieActionInstant;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.client.SparkRDDReadClient;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.SparkTaskContextSupplier;
+import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.HoodieCleanStat;
import org.apache.hudi.common.config.HoodieMetadataConfig;
@@ -52,13 +54,13 @@ import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
-import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -70,7 +72,6 @@ import org.apache.hudi.timeline.service.TimelineService;
import org.apache.hudi.util.JFunction;
import org.apache.hudi.utils.HoodieWriterClientTestHarness;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -107,6 +108,7 @@ import scala.Tuple2;
import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.apache.hudi.common.util.CleanerUtils.convertCleanMetadata;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertLinesMatch;
@@ -129,9 +131,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
protected JavaSparkContext jsc;
protected HoodieSparkEngineContext context;
protected SparkSession sparkSession;
- protected StorageConfiguration<Configuration> storageConf;
protected SQLContext sqlContext;
- protected HoodieStorage storage;
protected ExecutorService executorService;
protected HoodieTableMetaClient metaClient;
protected SparkRDDWriteClient writeClient;
@@ -398,6 +398,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
return readClient;
}
+ @Override
public SparkRDDWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
if (null != writeClient) {
writeClient.close();
@@ -472,6 +473,22 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
return Pair.of(partitionPathStatMap, globalStat);
}
+ @Override
+ protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) {
+ client.startCommitWithTime(commitTime);
+ JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts, 2);
+ JavaRDD<WriteStatus> statusRDD = ((SparkRDDWriteClient) client).upsert(insertRecordsRDD1, commitTime);
+ if (autoCommitOff) {
+ client.commit(commitTime, statusRDD);
+ }
+ List<WriteStatus> statuses = statusRDD.collect();
+ assertNoWriteErrors(statuses);
+ verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig(),
+ HoodieSparkKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
+
+ return statuses;
+ }
+
/**
* Validate the metadata tables contents to ensure it matches what is on the file system.
*/
@@ -683,10 +700,6 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
return new HoodieInstant(inflightOnly, "clean", instantTime);
}
- protected HoodieTableMetaClient createMetaClient(String basePath) {
- return HoodieTestUtils.createMetaClient(storageConf, basePath);
- }
-
protected HoodieTableMetaClient createMetaClient(SparkSession spark, String basePath) {
return HoodieClientTestUtils.createMetaClient(spark, basePath);
}
diff --git a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
index a2786e0a179..4a2768119bf 100644
--- a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
+++ b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
@@ -22,7 +22,6 @@ import org.apache.flink.test.util.AbstractTestBase;
import org.apache.flink.types.Row;
import org.apache.hudi.common.model.HoodieTableType;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
index e536e0c0853..87ce14f5c84 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.testutils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -26,6 +27,8 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
@@ -46,6 +49,9 @@ public class HoodieCommonTestHarness {
@TempDir
public java.nio.file.Path tempDir;
+ protected StorageConfiguration<Configuration> storageConf;
+ protected HoodieStorage storage;
+
protected void setTableName(String tableName) {
this.tableName = tableName;
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
index 4192a47d51d..1c747962cf6 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
@@ -54,6 +54,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.table.HoodieTableConfig.TYPE;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -156,7 +157,7 @@ public class TestDataSourceReadWithDeletes extends SparkClientFunctionalTestHarn
metaClient = HoodieTableMetaClient.reload(metaClient);
client.startCommitWithTime(instant);
List<WriteStatus> writeStatuses = client.upsert(writeRecords, instant).collect();
- org.apache.hudi.testutils.Assertions.assertNoWriteErrors(writeStatuses);
+ assertNoWriteErrors(writeStatuses);
metaClient = HoodieTableMetaClient.reload(metaClient);
return writeStatuses;
}