You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2024/06/04 09:21:44 UTC

(hudi) branch master updated: [HUDI-7704] Unify test client storage classes with duplicate code (#11152)

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 06457c0aff2 [HUDI-7704] Unify test client storage classes with duplicate code (#11152)
06457c0aff2 is described below

commit 06457c0aff2a102a0c760e60158280004914c0ca
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Tue Jun 4 16:21:37 2024 +0700

    [HUDI-7704] Unify test client storage classes with duplicate code (#11152)
    
    Co-authored-by: Vova Kolmakov <ko...@huawei-partners.com>
---
 .../java/org/apache/hudi/testutils/Assertions.java |   97 ++
 .../hudi/utils/HoodieWriterClientTestHarness.java  | 1297 +++++++++++++-
 .../hudi/client/TestJavaHoodieBackedMetadata.java  |   21 +-
 .../TestHoodieJavaClientOnCopyOnWriteStorage.java  | 1421 ++-------------
 .../testutils/HoodieJavaClientTestHarness.java     |   31 +-
 .../hudi/client/TestTableSchemaEvolution.java      |    2 +-
 .../functional/TestHoodieBackedMetadata.java       |   21 +-
 .../TestHoodieClientOnCopyOnWriteStorage.java      | 1824 +++-----------------
 .../hudi/testutils/HoodieClientTestBase.java       |    3 +
 .../testutils/HoodieSparkClientTestHarness.java    |   29 +-
 .../quickstart/TestHoodieFlinkQuickstart.java      |    1 -
 .../common/testutils/HoodieCommonTestHarness.java  |    6 +
 .../apache/hudi/TestDataSourceReadWithDeletes.java |    3 +-
 13 files changed, 1780 insertions(+), 2976 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
index bb2ba84f8c9..f2d78c1eaec 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/Assertions.java
@@ -20,13 +20,28 @@
 package org.apache.hudi.testutils;
 
 import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodiePartitionMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordDelegate;
 import org.apache.hudi.common.testutils.CheckedFunction;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.RawTripTestPayload;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Commonly used assertion functions.
@@ -51,4 +66,86 @@ public class Assertions {
         assertEquals(fileSizeGetter.apply(status), status.getStat().getFileSizeInBytes())));
   }
 
+  public static void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords,
+                                                HoodieStorage storage) throws IOException {
+    String[] partitionPathSet = inputRecords.stream()
+            .map(HoodieRecord::getPartitionPath).distinct().toArray(String[]::new);
+    assertPartitionMetadata(basePath, partitionPathSet, storage);
+  }
+
+  public static void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys,
+                                                    HoodieStorage storage) throws IOException {
+    String[] partitionPathSet = inputKeys.stream()
+            .map(HoodieKey::getPartitionPath).distinct().toArray(String[]::new);
+    assertPartitionMetadata(basePath, partitionPathSet, storage);
+  }
+
+  /**
+   * Ensure presence of partition meta-data at known depth.
+   *
+   * @param partitionPaths Partition paths to check
+   * @param storage        {@link HoodieStorage} instance.
+   * @throws IOException in case of error
+   */
+  public static void assertPartitionMetadata(String basePath, String[] partitionPaths,
+                                             HoodieStorage storage) throws IOException {
+    for (String partitionPath : partitionPaths) {
+      assertTrue(
+              HoodiePartitionMetadata.hasPartitionMetadata(
+                      storage, new StoragePath(basePath, partitionPath)));
+      HoodiePartitionMetadata pmeta =
+              new HoodiePartitionMetadata(storage, new StoragePath(basePath, partitionPath));
+      pmeta.readFromFS();
+      assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth());
+    }
+  }
+
+  /**
+   * Assert that there is no duplicate key at the partition level.
+   *
+   * @param records List of Hoodie records
+   */
+  public static void assertNoDupesWithinPartition(List<HoodieRecord<RawTripTestPayload>> records) {
+    Map<String, Set<String>> partitionToKeys = new HashMap<>();
+    for (HoodieRecord r : records) {
+      String key = r.getRecordKey();
+      String partitionPath = r.getPartitionPath();
+      if (!partitionToKeys.containsKey(partitionPath)) {
+        partitionToKeys.put(partitionPath, new HashSet<>());
+      }
+      assertFalse(partitionToKeys.get(partitionPath).contains(key), "key " + key + " is duplicate within partition " + partitionPath);
+      partitionToKeys.get(partitionPath).add(key);
+    }
+  }
+
+  /**
+   * Assert that there is no duplicate key at the partition level.
+   *
+   * @param recordDelegates List of Hoodie record delegates
+   */
+  public static void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
+    Map<String, Set<String>> partitionToKeys = new HashMap<>();
+    for (HoodieRecordDelegate r : recordDelegates) {
+      String recordKey = r.getRecordKey();
+      String partitionPath = r.getPartitionPath();
+      if (!partitionToKeys.containsKey(partitionPath)) {
+        partitionToKeys.put(partitionPath, new HashSet<>());
+      }
+      assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
+      partitionToKeys.get(partitionPath).add(recordKey);
+    }
+  }
+
+  public static void assertActualAndExpectedPartitionPathRecordKeyMatches(List<Pair<String, String>> expectedPartitionPathRecKeyPairs,
+                                                                    List<Pair<String, String>> actualPartitionPathRecKeyPairs) {
+    // verify all partitionpath, record key matches
+    assertEquals(expectedPartitionPathRecKeyPairs.size(), actualPartitionPathRecKeyPairs.size());
+    for (Pair<String, String> entry : actualPartitionPathRecKeyPairs) {
+      assertTrue(expectedPartitionPathRecKeyPairs.contains(entry));
+    }
+
+    for (Pair<String, String> entry : expectedPartitionPathRecKeyPairs) {
+      assertTrue(actualPartitionPathRecKeyPairs.contains(entry));
+    }
+  }
 }
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
index 2d4e87c52e6..0d2944bf408 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
@@ -18,44 +18,126 @@
 
 package org.apache.hudi.utils;
 
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.client.BaseHoodieWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
 import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.config.LockConfiguration;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
+import org.apache.hudi.common.engine.HoodieEngineContext;
 import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieAvroRecord;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
+import org.apache.hudi.common.model.HoodieFileGroupId;
 import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodiePartitionMetadata;
+import org.apache.hudi.common.model.HoodieOperation;
+import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
 import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.marker.MarkerType;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
 import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
 import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
 import org.apache.hudi.common.testutils.RawTripTestPayload;
+import org.apache.hudi.common.util.ClusteringUtils;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.FileFormatUtils;
+import org.apache.hudi.common.util.MarkerUtils;
+import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieCleanConfig;
+import org.apache.hudi.config.HoodieClusteringConfig;
 import org.apache.hudi.config.HoodieCompactionConfig;
 import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieLockConfig;
+import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieCommitException;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.keygen.KeyGenerator;
+import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
 import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.HoodieWriteMetadata;
+import org.apache.hudi.table.action.commit.HoodieWriteHelper;
+import org.apache.hudi.table.marker.WriteMarkersFactory;
 import org.apache.hudi.testutils.MetadataMergeWriteStatus;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
+import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
 import static org.apache.hudi.common.testutils.HoodieTestUtils.RAW_TRIPS_TEST_NAME;
+import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
+import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
+import static org.apache.hudi.config.HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING;
+import static org.apache.hudi.testutils.Assertions.assertNoDupesWithinPartition;
+import static org.apache.hudi.testutils.Assertions.assertNoDuplicatesInPartition;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
-public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
+public abstract class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
   protected static int timelineServicePort = FileSystemViewStorageConfig.REMOTE_PORT_NUM.defaultValue();
+  protected static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
+
+  protected HoodieTestTable testTable;
+
+  protected abstract BaseHoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg);
 
   protected void addConfigsForPopulateMetaFields(HoodieWriteConfig.Builder configBuilder, boolean populateMetaFields,
                                                  boolean isMetadataTable) {
@@ -76,8 +158,8 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
   public static Properties getPropertiesForKeyGen(boolean populateMetaFields) {
     Properties properties = new Properties();
     properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields));
-    properties.put("hoodie.datasource.write.recordkey.field", "_row_key");
-    properties.put("hoodie.datasource.write.partitionpath.field", "partition_path");
+    properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+    properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
     properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key");
     properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path");
     return properties;
@@ -86,7 +168,7 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
   protected Properties getPropertiesForMetadataTable() {
     Properties properties = new Properties();
     properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false");
-    properties.put("hoodie.datasource.write.recordkey.field", "key");
+    properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "key");
     properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "key");
     return properties;
   }
@@ -165,71 +247,1182 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
     return builder;
   }
 
-  public void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords,
-                                                HoodieStorage storage) throws IOException {
-    Set<String> partitionPathSet = inputRecords.stream()
-        .map(HoodieRecord::getPartitionPath)
-        .collect(Collectors.toSet());
-    assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
+  // Functional Interfaces for passing lambda and Hoodie Write API contexts
+
+  @FunctionalInterface
+  public interface Function2<R, T1, T2> {
+
+    R apply(T1 v1, T2 v2) throws IOException;
+  }
+
+  @FunctionalInterface
+  public interface Function3<R, T1, T2, T3> {
+
+    R apply(T1 v1, T2 v2, T3 v3) throws IOException;
+  }
+
+  /* Auxiliary methods for testing CopyOnWriteStorage with Spark and Java clients
+  to avoid code duplication in TestHoodieClientOnCopyOnWriteStorage and TestHoodieJavaClientOnCopyOnWriteStorage */
+
+  protected abstract  List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff)
+      throws IOException;
+
+  protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                        String initCommitTime, int numRecordsInThisCommit,
+                                        Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                        boolean assertForCommit, int expRecordsInThisCommit) throws Exception {
+    return castInsertFirstBatch(writeConfig, client, newCommitTime, initCommitTime, numRecordsInThisCommit, writeFn,
+        isPreppedAPI, assertForCommit, expRecordsInThisCommit, true);
+  }
+
+  protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                        String initCommitTime, int numRecordsInThisCommit,
+                                        Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                        boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+    return null; // override in subclasses if needed
+  }
+
+  protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                  Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                  Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+                                  Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+                                  boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit) throws Exception {
+    return castWriteBatch(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+        writeFn, assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, true);
+  }
+
+  protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                  Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                  Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+                                  Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+                                  boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+                                  boolean filterForCommitTimeWithAssert) throws Exception {
+    return null; // override in subclasses if needed
+  }
+
+  protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                   Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                   Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+                                   boolean filterForCommitTimeWithAssert) throws Exception {
+    return null; // override in subclasses if needed
+  }
+
+  protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                   String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, boolean assertForCommit,
+                                   int expRecordsInThisCommit, int expTotalRecords) throws Exception {
+    return castDeleteBatch(writeConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit, isPreppedAPI,
+        assertForCommit, expRecordsInThisCommit, expTotalRecords, true);
+  }
+
+  protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                   String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+    return null; // override in subclasses if needed
+  }
+
+  protected Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI, HoodieWriteConfig writeConfig,
+                                                                                 Function2<List<HoodieRecord>, String, Integer> wrapped) {
+    return null; // override in subclasses if needed
+  }
+
+  protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+    return new String[0]; // override in subclasses if needed
+  }
+
+  protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
+    // override in subclasses if needed
+  }
+
+  protected HoodieWriteConfig getRollbackMarkersAndConsistencyGuardWriteConfig(
+          boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard, boolean populateMetaFields) {
+    Properties properties = new Properties();
+    if (!populateMetaFields) {
+      properties = getPropertiesForKeyGen();
+    }
+
+    return !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
+            .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+                    .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1)
+                    .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
+            .build() :
+            getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
+                    .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
+                            .withConsistencyCheckEnabled(true)
+                            .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
+                            .withOptimisticConsistencyGuardSleepTimeMs(1).build())
+                    .withProperties(properties).build();
+  }
+
+  protected HoodieWriteConfig getConsistencyCheckWriteConfig(boolean enableOptimisticConsistencyGuard) {
+    return !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
+            .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+                    .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
+            .build()) : (getConfigBuilder().withAutoCommit(false)
+            .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
+                    .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
+                    .withOptimisticConsistencyGuardSleepTimeMs(1).build())
+            .build());
+  }
+
+  protected HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
+    Properties properties = new Properties();
+    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
+    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
+    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
+    if (!populateMetaFields) {
+      properties.putAll(getPropertiesForKeyGen(populateMetaFields));
+    }
+    return getConfigBuilder()
+            .withEmbeddedTimelineServerEnabled(false)
+            .withCleanConfig(HoodieCleanConfig.newBuilder()
+                    .withFailedWritesCleaningPolicy(cleaningPolicy)
+                    .withAutoClean(false).build())
+            .withTimelineLayoutVersion(1)
+            .withHeartbeatIntervalInMs(3 * 1000)
+            .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+                    .withRemoteServerPort(timelineServicePort).build())
+            .withAutoCommit(false)
+            .withLockConfig(HoodieLockConfig.newBuilder()
+                    .withLockProvider(InProcessLockProvider.class)
+                    .build())
+            .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
+            .withProperties(properties).build();
+  }
+
+  @NotNull
+  protected Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
+    for (WriteStatus status : allStatus) {
+      StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
+      records.addAll(getFileUtilsInstance(createMetaClient()).readAvroRecords(storage, filePath));
+    }
+    Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
+    assertEquals(records.size(), expectedKeys.size());
+    return expectedKeys;
+  }
+
+  protected void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
+    List<GenericRecord> records = new ArrayList<>();
+    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
+    Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
+            .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
+    assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
+    Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
+    for (GenericRecord record : records) {
+      String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+      assertTrue(expectedKeys.contains(recordKey));
+      String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
+      assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
+    }
+  }
+
+  protected Pair<String, String> getPartitionAndBaseFilePathsFromLatestCommitMetadata(HoodieTableMetaClient metaClient) throws IOException {
+    String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
+    HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+        .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
+            metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
+            HoodieCommitMetadata.class);
+    String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
+        .flatMap(Collection::stream).filter(s -> s.getPath().endsWith(extension)).findAny()
+        .map(HoodieWriteStat::getPath).orElse(null);
+    String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
+        .flatMap(Collection::stream).filter(s -> s.getPath().endsWith(extension)).findAny()
+        .map(HoodieWriteStat::getPartitionPath).orElse(null);
+    return Pair.of(partitionPath, metaClient.getBasePathV2().toString() + StoragePath.SEPARATOR + filePath);
   }
 
-  public void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys,
-                                             HoodieStorage storage) throws IOException {
-    Set<String> partitionPathSet = inputKeys.stream()
-        .map(HoodieKey::getPartitionPath)
-        .collect(Collectors.toSet());
-    assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
+  public static FileFormatUtils getFileUtilsInstance(HoodieTableMetaClient metaClient) {
+    return HoodieIOFactory.getIOFactory(metaClient.getStorage())
+            .getFileFormatUtils(metaClient.getTableConfig().getBaseFileFormat());
   }
 
   /**
-   * Ensure presence of partition meta-data at known depth.
-   *
-   * @param partitionPaths Partition paths to check
-   * @param storage        {@link HoodieStorage} instance.
-   * @throws IOException in case of error
+   * Build Hoodie Write Config for small data file sizes.
+   */
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, boolean mergeAllowDuplicateInserts) {
+    return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, dataGen.getEstimatedFileSizeInBytes(150), mergeAllowDuplicateInserts);
+  }
+
+  /**
+   * Build Hoodie Write Config for specified small file sizes.
    */
-  public static void assertPartitionMetadata(String basePath, String[] partitionPaths,
-                                             HoodieStorage storage) throws IOException {
-    for (String partitionPath : partitionPaths) {
-      assertTrue(
-          HoodiePartitionMetadata.hasPartitionMetadata(
-              storage, new StoragePath(basePath, partitionPath)));
-      HoodiePartitionMetadata pmeta =
-          new HoodiePartitionMetadata(storage, new StoragePath(basePath, partitionPath));
-      pmeta.readFromFS();
-      assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth());
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+    String schemaStr = useNullSchema ? NULL_SCHEMA : TRIP_EXAMPLE_SCHEMA;
+    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts);
+  }
+
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize) {
+    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false);
+  }
+
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts, true, new Properties());
+  }
+
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
+    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
+  }
+
+  protected HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
+                                                      boolean populateMetaFields, Properties props) {
+    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
+    if (!populateMetaFields) {
+      builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.SIMPLE).build());
     }
+    return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
+                    .compactionSmallFileSize(smallFileSize)
+                    // Set rollback to LAZY so no inflights are deleted
+                    .insertSplitSize(insertSplitSize).build())
+            .withCleanConfig(HoodieCleanConfig.newBuilder()
+                    .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+                    .build())
+            .withStorageConfig(HoodieStorageConfig.newBuilder()
+                    .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
+                    .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
+            .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
+            .withProps(props)
+            .build();
   }
 
-  // Functional Interfaces for passing lambda and Hoodie Write API contexts
+  protected HoodieWriteConfig getSmallInsertWriteConfigForMDT(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
+    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
+    return builder.withCompactionConfig(
+                    HoodieCompactionConfig.newBuilder()
+                            .compactionSmallFileSize(smallFileSize)
+                            // Set rollback to LAZY so no inflights are deleted
+                            .insertSplitSize(insertSplitSize).build())
+            .withCleanConfig(HoodieCleanConfig.newBuilder()
+                    .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
+            .withStorageConfig(
+                    HoodieStorageConfig.newBuilder()
+                            .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
+                            .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
+            .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
+            .build();
+  }
 
-  @FunctionalInterface
-  public interface Function2<R, T1, T2> {
+  protected HoodieWriteConfig getWriteConfigWithPopulateMetaFieldsAndAllowOperationMetaField(boolean populateMetaFields, boolean allowOperationMetadataField) {
+    HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
+            .withAllowOperationMetadataField(allowOperationMetadataField).combineInput(true, true);
+    addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
+    return configBuilder.build();
+  }
 
-    R apply(T1 v1, T2 v2) throws IOException;
+  protected List<HoodieRecord<RawTripTestPayload>> dedupForCopyOnWriteStorage(HoodieWriteConfig writeConfig, HoodieData<HoodieRecord> records,
+                                                                              boolean isGlobal, int additionalParallelism, int expectedNumPartitions) {
+    HoodieIndex index = mock(HoodieIndex.class);
+    when(index.isGlobal()).thenReturn(isGlobal);
+    int dedupParallelism = records.getNumPartitions() + additionalParallelism;
+    HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd = (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
+            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
+    assertEquals(expectedNumPartitions, dedupedRecsRdd.getNumPartitions());
+    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
+    assertEquals(isGlobal ? 1 : 2, dedupedRecs.size());
+    assertNoDupesWithinPartition(dedupedRecs);
+    return dedupedRecs;
   }
 
-  @FunctionalInterface
-  public interface Function3<R, T1, T2, T3> {
+  protected HoodieTableMetaClient createMetaClient() {
+    return HoodieTestUtils.createMetaClient(storageConf, basePath);
+  }
 
-    R apply(T1 v1, T2 v2, T3 v3) throws IOException;
+  protected HoodieTableMetaClient createMetaClient(String basePath) {
+    return HoodieTestUtils.createMetaClient(storageConf, basePath);
+  }
+
+  protected Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
+    return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
   }
 
   /**
-   * Assert that there is no duplicate key at the partition level.
+   * Find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
    *
-   * @param records List of Hoodie records
-   */
-  public static void assertNodupesWithinPartition(List<HoodieRecord<RawTripTestPayload>> records) {
-    Map<String, Set<String>> partitionToKeys = new HashMap<>();
-    for (HoodieRecord r : records) {
-      String key = r.getRecordKey();
-      String partitionPath = r.getPartitionPath();
-      if (!partitionToKeys.containsKey(partitionPath)) {
-        partitionToKeys.put(partitionPath, new HashSet<>());
+   * @param partitionPath where parquet files created as part of clustering
+   * @throws IOException
+   */
+  protected void verifyClusteredFilesWithReplaceCommitMetadata(String partitionPath) throws IOException {
+    metaClient = HoodieTableMetaClient.reload(createMetaClient());
+    HoodieInstant replaceCommitInstant =
+            metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
+    HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
+            .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
+                    HoodieReplaceCommitMetadata.class);
+
+    List<String> filesFromReplaceCommit = new ArrayList<>();
+    replaceCommitMetadata.getPartitionToWriteStats()
+            .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
+
+    // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
+    List<StoragePathInfo> pathInfoList =
+            storage.listDirectEntries(new StoragePath(basePath, partitionPath));
+    List<String> clusteredFiles = pathInfoList.stream()
+            .filter(entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
+            .map(pathInfo -> partitionPath + StoragePath.SEPARATOR + pathInfo.getPath().getName())
+            .collect(Collectors.toList());
+    assertEquals(clusteredFiles, filesFromReplaceCommit);
+  }
+
+  protected static List<Pair<HoodieInstant, HoodieClusteringPlan>> getAndAssertPendingClusteringPlans(boolean scheduleInlineClustering, HoodieTableMetaClient metaClient) {
+    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
+            ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
+    if (scheduleInlineClustering) {
+      assertEquals(1, pendingClusteringPlans.size());
+    } else {
+      assertEquals(0, pendingClusteringPlans.size());
+    }
+    return  pendingClusteringPlans;
+  }
+
+  protected void checkTimelineForUpsertsInternal(HoodieTableMetaClient metaClient) {
+    HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
+    List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
+    assertEquals(5, instants.size());
+    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"), instants.get(0));
+    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"), instants.get(1));
+    // New Format should have all states of instants
+    assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"), instants.get(2));
+    assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"), instants.get(3));
+    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"), instants.get(4));
+  }
+
+  /**
+   * Verify data in base files matches expected records and commit time.
+   */
+  protected void verifyRecordsWritten(String commitTime, boolean populateMetadataField, List<HoodieRecord> expectedRecords,
+                                    List<WriteStatus> allStatus, HoodieWriteConfig config, KeyGenerator keyGenerator) {
+    List<GenericRecord> records = new ArrayList<>();
+    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
+    if (config.populateMetaFields()) {
+      for (GenericRecord record : records) {
+        String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+        assertEquals(commitTime,
+            record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
+        assertTrue(expectedKeys.contains(recordKey));
+      }
+    } else {
+      for (GenericRecord record : records) {
+        String recordKey = keyGenerator.getKey(record).getRecordKey();
+        if (!populateMetadataField) {
+          assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
+        }
+        assertTrue(expectedKeys.contains(recordKey));
+      }
+    }
+  }
+
+  protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
+    return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
+  }
+
+  /**
+   * This method returns following three items:
+   * 1. List of all HoodieRecord written in the two batches of insert.
+   * 2. Commit instants of the two batches.
+   * 3. List of new file group ids that were written in the two batches.
+   */
+  protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> insertTwoBatches(BaseHoodieWriteClient writeClient, BaseHoodieWriteClient brokenClusteringClient,
+                                                                                                  boolean populateMetaFields, String partitionPath, boolean failInlineClustering) throws IOException {
+    BaseHoodieWriteClient client;
+    if (failInlineClustering) {
+      if (null != writeClient) {
+        writeClient.close();
+      }
+      client = brokenClusteringClient;
+    } else {
+      client = writeClient;
+    }
+
+    dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
+    String commitTime1 = client.createNewInstantTime();
+    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
+    List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
+    Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
+
+    String commitTime2 = client.createNewInstantTime();
+    List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
+    List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
+    client.close();
+    Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
+    Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
+    fileIdsUnion.addAll(fileIds2);
+    //verify new files are created for 2nd write
+    Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
+    fileIdIntersection.retainAll(fileIds2);
+    assertEquals(0, fileIdIntersection.size());
+    return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
+  }
+
+  protected void rollbackAndAssert(boolean enableOptimisticConsistencyGuard, String instantTime, HoodieTableMetaClient metaClient, BaseHoodieWriteClient client) throws IOException {
+    if (!enableOptimisticConsistencyGuard) {
+      // Rollback of this commit should succeed with FailSafeCG
+      client.rollback(instantTime);
+      assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present");
+      // Marker directory must be removed after rollback
+      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+    } else {
+      // if optimistic CG is enabled, commit should have succeeded.
+      assertTrue(testTable.commitExists(instantTime),
+              "With optimistic CG, first commit should succeed. commit file should be present");
+      // Marker directory must be removed after rollback
+      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+      client.rollback(instantTime);
+      assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present");
+    }
+  }
+
+  protected Pair<StoragePath, List<WriteStatus>> testConsistencyCheck(HoodieEngineContext context, HoodieTableMetaClient metaClient,
+                                                                    String instantTime, boolean enableOptimisticConsistencyGuard,
+                                                                    Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+                                                                    Function transformInputFn, Function transformOutputFn)
+          throws Exception {
+    HoodieWriteConfig cfg = getConsistencyCheckWriteConfig(enableOptimisticConsistencyGuard);
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+
+    client.startCommitWithTime(instantTime);
+    List<HoodieRecord> writeRecords = dataGen.generateInserts(instantTime, 200);
+    List<WriteStatus> result = (List<WriteStatus>) transformOutputFn.apply(client.bulkInsert(transformInputFn.apply(writeRecords), instantTime));
+
+    // Create a dummy marker file to simulate the case that a marker file was created without data file.
+    // This should fail the commit
+    String partitionPath;
+    String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
+    if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
+      String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
+                      markerFolderPath, storage, context, 1).values().stream()
+              .flatMap(Collection::stream).findFirst().get();
+      partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
+    } else {
+      partitionPath = storage.globEntries(
+                      new StoragePath(String.format("%s/*/*/*/*", markerFolderPath)), path ->
+                              path.toString().contains(HoodieTableMetaClient.MARKER_EXTN))
+              .stream()
+              .limit(1).map(status -> status.getPath().getParent().toString())
+              .collect(Collectors.toList()).get(0);
+    }
+
+    Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
+                    cfg.getMarkersType(), getHoodieTableFn.apply(metaClient, cfg), instantTime)
+            .create(partitionPath,
+                    FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(),
+                            HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()),
+                    IOType.MERGE);
+    if (!enableOptimisticConsistencyGuard) {
+      Exception e = assertThrows(HoodieCommitException.class, () -> client.commit(instantTime, transformInputFn.apply(result)),
+              "Commit should fail due to consistency check");
+      assertTrue(e.getCause() instanceof HoodieIOException);
+    } else {
+      // with optimistic CG, commit should succeed
+      client.commit(instantTime, transformInputFn.apply(result));
+    }
+    return Pair.of(markerFilePath.get(), result);
+  }
+
+  protected void testConsistencyCheckDuringFinalize(HoodieEngineContext context, boolean enableOptimisticConsistencyGuard,
+                                                    Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+                                                    Function transformInputFn, Function transformOutputFn) throws Exception {
+    HoodieTableMetaClient metaClient = createMetaClient();
+    String instantTime = "000";
+    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
+            .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+    Pair<StoragePath, List<WriteStatus>> result = testConsistencyCheck(context, metaClient, instantTime,
+              enableOptimisticConsistencyGuard, getHoodieTableFn, transformInputFn, transformOutputFn);
+
+    // Delete orphan marker and commit should succeed
+    metaClient.getStorage().deleteFile(result.getKey());
+    if (!enableOptimisticConsistencyGuard) {
+      assertTrue(client.commit(instantTime, transformInputFn.apply(result.getRight())), "Commit should succeed");
+      assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+      // Marker directory must be removed
+      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+    } else {
+      // with optimistic, first client.commit should have succeeded.
+      assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+      // Marker directory must be removed
+      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
+    }
+  }
+
+  protected void testRollbackAfterConsistencyCheckFailureUsingFileList(HoodieEngineContext context, boolean rollbackUsingMarkers,
+                                                                       boolean enableOptimisticConsistencyGuard, boolean populateMetaFields,
+                                                                       Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTableFn,
+                                                                       Function transformInputFn, Function transformOutputFn) throws Exception {
+    String instantTime = "00000000000010";
+    HoodieTableMetaClient metaClient = createMetaClient();
+    HoodieWriteConfig cfg = getRollbackMarkersAndConsistencyGuardWriteConfig(rollbackUsingMarkers, enableOptimisticConsistencyGuard, populateMetaFields);
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+    testConsistencyCheck(context, metaClient, instantTime, enableOptimisticConsistencyGuard, getHoodieTableFn, transformInputFn, transformOutputFn);
+    rollbackAndAssert(enableOptimisticConsistencyGuard, instantTime, metaClient, client);
+  }
+
+  protected List<HoodieRecord> generate3AvroRecords(String commitTime) throws IOException {
+    String recordKey = UUID.randomUUID().toString();
+    HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
+    HoodieRecord<RawTripTestPayload> recordOne =
+            new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, commitTime), HoodieOperation.INSERT);
+    HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
+    HoodieRecord recordTwo =
+            new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, commitTime), HoodieOperation.INSERT);
+    // Same key and partition as keyTwo
+    HoodieRecord recordThree =
+            new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, commitTime), HoodieOperation.UPDATE_AFTER);
+    return Arrays.asList(recordOne, recordTwo, recordThree);
+  }
+
+  private HoodieWriteMetadata<List<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
+                                                                   boolean populateMetaFields,
+                                                                   boolean completeClustering,
+                                                                   String validatorClasses,
+                                                                   String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
+                                                                   Pair<List<HoodieRecord>, List<String>> allRecords,
+                                                                   Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> transformWriteMetadataFn,
+                                                                   Function<HoodieWriteConfig, KeyGenerator> createKeyGeneratorFn) {
+    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
+            .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
+            .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
+            .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
+            .build();
+
+    HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
+            .withPreCommitValidatorConfig(validatorConfig)
+            .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
+            .withClusteringConfig(clusteringConfig).build();
+
+    // create client with new config.
+    BaseHoodieWriteClient client = getHoodieWriteClient(config);
+    String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
+    HoodieWriteMetadata<List<WriteStatus>> clusterMetadata = transformWriteMetadataFn.apply(client.cluster(clusteringCommitTime, completeClustering));
+    if (config.populateMetaFields()) {
+      verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses());
+    } else {
+      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses(),
+              config, createKeyGeneratorFn.apply(config));
+    }
+    return clusterMetadata;
+  }
+
+  protected void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
+                                String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
+                                Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords,
+                                Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> transformWriteMetadataFn,
+                                Function<HoodieWriteConfig, KeyGenerator> createKeyGeneratorFn) {
+
+    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
+            .withClusteringConfig(clusteringConfig)
+            .withProps(getPropertiesForKeyGen()).build();
+    HoodieWriteMetadata<List<WriteStatus>> clusterMetadata =
+            performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation,
+                    sqlQueryForSingleResultValidation, allRecords.getLeft(), transformWriteMetadataFn, createKeyGeneratorFn);
+    if (assertSameFileIds) {
+      Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
+              .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
+      Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
+      assertEquals(insertedFileIds, replacedFileIds);
+    }
+    if (completeClustering) {
+      String clusteringCommitTime = createMetaClient().reloadActiveTimeline().getCompletedReplaceTimeline()
+              .getReverseOrderedInstants().findFirst().get().getTimestamp();
+      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(),
+              clusterMetadata.getWriteStatuses(), config, createKeyGeneratorFn.apply(config));
+    }
+  }
+
+  protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, Function createBrokenClusteringClientFn) throws IOException {
+    return testInsertTwoBatches(populateMetaFields, "2015/03/16", createBrokenClusteringClientFn);
+  }
+
+  protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath,
+                                                                                                      Function createBrokenClusteringClientFn) throws IOException {
+    return testInsertTwoBatches(populateMetaFields, partitionPath, getPropertiesForKeyGen(populateMetaFields), false, createBrokenClusteringClientFn);
+  }
+
+  protected Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props, boolean failInlineClustering,
+                                                                                                      Function createBrokenClusteringClientFn) throws IOException {
+    // create config to not update small files.
+    HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
+            populateMetaFields ? props : getPropertiesForKeyGen());
+    return insertTwoBatches(getHoodieWriteClient(config), (BaseHoodieWriteClient) createBrokenClusteringClientFn.apply(config), populateMetaFields, partitionPath, failInlineClustering);
+  }
+
+  protected void testCommitWritesRelativePaths(Function transformInputFn) throws Exception {
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
+    addConfigsForPopulateMetaFields(cfgBuilder, true);
+    try (BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
+      HoodieTableMetaClient metaClient = createMetaClient();
+      String instantTime = "000";
+      client.startCommitWithTime(instantTime);
+      List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
+
+      assertTrue(client.commit(instantTime, client.bulkInsert(transformInputFn.apply(records), instantTime)), "Commit should succeed");
+      assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created");
+
+      // Get base file paths from commit metadata
+      String actionType = metaClient.getCommitActionType();
+      HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
+      HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
+      HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+              .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class);
+      String basePath = metaClient.getBasePathV2().toString();
+      Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
+
+      // Read from commit file
+      HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
+              metaClient.reloadActiveTimeline()
+                      .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
+              HoodieCommitMetadata.class);
+      HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
+      // Compare values in both to make sure they are equal.
+      for (String pathName : paths.values()) {
+        assertTrue(commitPathNames.contains(pathName));
+      }
+    }
+  }
+
+  protected void testMetadataStatsOnCommit(boolean populateMetaFields, Function transformInputFn) throws Exception {
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
+    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+    HoodieWriteConfig cfg = cfgBuilder.build();
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
+
+    String instantTime0 = "000";
+    client.startCommitWithTime(instantTime0);
+    List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
+    assertTrue(client.commit(instantTime0, client.bulkInsert(transformInputFn.apply(records0), instantTime0)), "Commit should succeed");
+    assertTrue(testTable.commitExists(instantTime0), "After explicit commit, commit file should be created");
+
+    // Read from commit file
+    HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
+            createMetaClient().reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
+            HoodieCommitMetadata.class);
+    int inserts = 0;
+    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+      for (HoodieWriteStat stat : pstat.getValue()) {
+        inserts += stat.getNumInserts();
+      }
+    }
+    assertEquals(200, inserts);
+
+    // Update + Inserts such that they just expand file1
+    String instantTime1 = "001";
+    client.startCommitWithTime(instantTime1);
+    List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
+    assertTrue(client.commit(instantTime1, client.upsert(transformInputFn.apply(records1), instantTime1)), "Commit should succeed");
+    assertTrue(testTable.commitExists(instantTime1), "After explicit commit, commit file should be created");
+
+    metaClient = createMetaClient();
+    // Read from commit file
+    metadata = HoodieCommitMetadata.fromBytes(
+            metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
+            HoodieCommitMetadata.class);
+    inserts = 0;
+    int upserts = 0;
+    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+      for (HoodieWriteStat stat : pstat.getValue()) {
+        inserts += stat.getNumInserts();
+        upserts += stat.getNumUpdateWrites();
       }
-      assertFalse(partitionToKeys.get(partitionPath).contains(key), "key " + key + " is duplicate within partition " + partitionPath);
-      partitionToKeys.get(partitionPath).add(key);
     }
+    assertEquals(0, inserts);
+    assertEquals(200, upserts);
+  }
+
+  protected void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail, Function createBrokenClusteringClientFn) throws IOException {
+    try {
+      Properties properties = new Properties();
+      properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
+      properties.setProperty("hoodie.auto.commit", "false");
+      properties.setProperty("hoodie.clustering.inline.max.commits", "1");
+      properties.setProperty("hoodie.clustering.inline", "true");
+      properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
+      testInsertTwoBatches(true, "2015/03/16", properties, true, createBrokenClusteringClientFn);
+      assertFalse(shouldFail);
+    } catch (HoodieException e) {
+      assertEquals(CLUSTERING_FAILURE, e.getMessage());
+      assertTrue(shouldFail);
+    }
+  }
+
+  protected void testDeletesWithoutInserts(boolean populateMetaFields, Function transformInputFn, Function transformOutputFn) {
+    final String testPartitionPath = "2016/09/26";
+    final int insertSplitLimit = 100;
+    // setup the small file handling params
+    HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
+            TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
+                    ? new Properties() : getPropertiesForKeyGen());
+    dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
+    BaseHoodieWriteClient client = getHoodieWriteClient(config);
+    // delete non existent keys
+    String commitTime1 = "001";
+    client.startCommitWithTime(commitTime1);
+
+    List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
+    List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
+    transformOutputFn.apply(client.delete(transformInputFn.apply(hoodieKeysToDelete), commitTime1));
+  }
+
+  protected void testInlineScheduleClustering(Function createBrokenClusteringClientFn, HoodieClusteringConfig clusteringConfig,
+                                              Function transformInputFn, Function transformOutputFn) throws IOException {
+    testInsertTwoBatches(true, createBrokenClusteringClientFn);
+    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
+        .withClusteringConfig(clusteringConfig)
+        .withProps(getPropertiesForKeyGen()).build();
+    dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
+    generateInsertsAndCommit(config, transformInputFn, transformOutputFn);
+    HoodieTableMetaClient metaClient = createMetaClient();
+    getAndAssertPendingClusteringPlans(clusteringConfig.getBoolean(SCHEDULE_INLINE_CLUSTERING), metaClient);
+  }
+
+  protected void testAndValidateClusteringOutputFiles(Function createBrokenClusteringClientFn, HoodieClusteringConfig clusteringConfig,
+                                              Function transformInputFn, Function transformOutputFn) throws IOException {
+    String partitionPath = "2015/03/16";
+    testInsertTwoBatches(true, partitionPath, createBrokenClusteringClientFn);
+    // Trigger clustering
+    HoodieWriteConfig config = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
+        .withClusteringConfig(clusteringConfig).build();
+    generateInsertsAndCommit(config, transformInputFn, transformOutputFn);
+    verifyClusteredFilesWithReplaceCommitMetadata(partitionPath);
+  }
+
+  private void generateInsertsAndCommit(HoodieWriteConfig config, Function transformInputFn, Function transformOutputFn) {
+    try (BaseHoodieWriteClient client = getHoodieWriteClient(config)) {
+      String commitTime = client.createNewInstantTime();
+      List<HoodieRecord> records = dataGen.generateInserts(commitTime, 200);
+      client.startCommitWithTime(commitTime);
+      List<WriteStatus> statusList = (List<WriteStatus>) transformOutputFn.apply(client.insert(transformInputFn.apply(records), commitTime));
+      assertNoWriteErrors(statusList);
+      client.commit(commitTime, transformInputFn.apply(statusList));
+    }
+  }
+
+  /**
+   * Test Deduplication Logic for write function.
+   *
+   * @param writeFn One of HoodieWriteClient non-prepped write APIs
+   * @throws Exception in case of failure
+   */
+  protected void testDeduplication(Function3<List<WriteStatus>, BaseHoodieWriteClient, List<HoodieRecord>, String> writeFn,
+                                   boolean populateMetaFields, boolean allowOperationMetadataField) throws Exception {
+    String newCommitTime = "001";
+    List<HoodieRecord> recordList = generate3AvroRecords(newCommitTime);
+    HoodieData<HoodieRecord> records = HoodieListData.eager(recordList);
+    HoodieWriteConfig writeConfig = getWriteConfigWithPopulateMetaFieldsAndAllowOperationMetaField(populateMetaFields, allowOperationMetadataField);
+    // Global dedup should be done based on recordKey only
+    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupForCopyOnWriteStorage(writeConfig, records, true, allowOperationMetadataField ? 100 : 2, records.getNumPartitions());
+    if (allowOperationMetadataField) {
+      assertEquals(dedupedRecs.get(0).getOperation(), recordList.get(2).getOperation());
+    } else {
+      assertEquals(dedupedRecs.get(0).getPartitionPath(), recordList.get(2).getPartitionPath());
+      // non-Global dedup should be done based on both recordKey and partitionPath
+      dedupForCopyOnWriteStorage(writeConfig, records, false, 2, records.getNumPartitions());
+    }
+    // Perform write-action and check
+    try (BaseHoodieWriteClient client = getHoodieWriteClient(writeConfig)) {
+      client.startCommitWithTime(newCommitTime);
+      List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
+      assertNoWriteErrors(statuses);
+      assertEquals(2, statuses.size());
+      assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
+              .collect(Collectors.toList()));
+    }
+  }
+
+  /**
+   * Test auto-commit by applying the write function.
+   *
+   * @param writeFn One of HoodieWriteClient Write API
+   * @throws Exception in case of failure
+   */
+  protected void testAutoCommit(Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+                              boolean isPrepped, boolean populateMetaFields) throws Exception {
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false); // Set autoCommit false
+    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+    try (BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
+      String prevCommitTime = "000";
+      String newCommitTime = "001";
+      int numRecords = 200;
+      Object result = castInsertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn, isPrepped, false, numRecords);
+      assertFalse(testTable.commitExists(newCommitTime), "If Autocommit is false, then commit should not be made automatically");
+      assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
+      assertTrue(testTable.commitExists(newCommitTime), "After explicit commit, commit file should be created");
+    }
+  }
+
+  /**
+   * When records getting inserted are deleted in the same write batch, hudi should have deleted those records and
+   * not be available in read path.
+   *
+   * @throws Exception
+   */
+  protected void testDeletesForInsertsInSameBatch() throws Exception {
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
+    addConfigsForPopulateMetaFields(cfgBuilder, true);
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build());
+    /**
+     * Write 200 inserts and issue deletes to a subset(50) of inserts.
+     */
+    String initCommitTime = "000";
+    String newCommitTime = "001";
+
+    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
+    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        (String instantTime, Integer numRecordsInThisCommit) -> {
+          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
+          List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
+
+          recordsInFirstBatch.addAll(fewRecordsForInsert);
+          recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+          return recordsInFirstBatch;
+        };
+
+    castWriteBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime, -1, recordGenFunction,
+        BaseHoodieWriteClient::upsert, true, 150, 150, 1, false, true);
+  }
+
+  /**
+   * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
+   *
+   * @throws Exception in case of error
+   */
+  protected void testHoodieConcatHandle(boolean populateMetaFields, boolean isPrepped) throws Exception {
+    metaClient = createMetaClient(basePath);
+    // Force using older timeline layout
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
+    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+    HoodieWriteConfig hoodieWriteConfig = cfgBuilder.withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(VERSION_0).build();
+    HoodieTableMetaClient.withPropertyBuilder().fromMetaClient(metaClient).setTimelineLayoutVersion(VERSION_0)
+        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
+
+    BaseHoodieWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
+
+    // Write 1 (only inserts)
+    String newCommitTime = "001";
+    String initCommitTime = "000";
+    int numRecords = 200;
+    castInsertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, BaseHoodieWriteClient::insert,
+        isPrepped, true, numRecords, populateMetaFields);
+
+    // Write 2 (updates)
+    String prevCommitTime = newCommitTime;
+    newCommitTime = "004";
+    numRecords = 100;
+    String commitTimeBetweenPrevAndNew = "002";
+
+    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
+
+    castWriteBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, recordGenFunction,
+        BaseHoodieWriteClient::insert, true, numRecords, 300, 2, false, populateMetaFields);
+  }
+
+  protected void testHoodieConcatHandleOnDupInserts(boolean isPrepped) throws Exception {
+    HoodieWriteConfig config = getConfigBuilder()
+        .withMergeAllowDuplicateOnInserts(true)
+        .build();
+
+    BaseHoodieWriteClient client = getHoodieWriteClient(config);
+
+    // Write 1 (only inserts)
+    String initCommitTime = "000";
+    String newCommitTime = "001";
+    int firstInsertRecords = 50;
+    castInsertFirstBatch(config, client, newCommitTime, initCommitTime, firstInsertRecords, BaseHoodieWriteClient::insert,
+        isPrepped, true, firstInsertRecords, config.populateMetaFields());
+
+    // Write 2 (updates with duplicates)
+    String prevCommitTime = newCommitTime;
+    newCommitTime = "004";
+    int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
+    List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
+
+    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        generateWrapRecordsFn(isPrepped, config, dataGen::generateUpdates);
+
+    castWriteBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
+        secondInsertRecords, recordGenFunction, BaseHoodieWriteClient::insert, true, secondInsertRecords,
+        firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+  }
+
+  /**
+   * Test one of HoodieWriteClient upsert(Prepped) APIs.
+   *
+   * @param writeFn One of Hoodie Write Function API
+   * @throws Exception in case of error
+   */
+  protected void testUpsertsInternal(Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean populateMetaFields, boolean isPrepped) throws Exception {
+    metaClient = createMetaClient();
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withRollbackUsingMarkers(true);
+    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+    // Force using older timeline layout
+    HoodieWriteConfig config = cfgBuilder.withTimelineLayoutVersion(VERSION_0).build();
+    HoodieTableMetaClient.withPropertyBuilder().fromMetaClient(metaClient).setTimelineLayoutVersion(VERSION_0)
+        .setPopulateMetaFields(populateMetaFields).initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
+
+    BaseHoodieWriteClient client = getHoodieWriteClient(config);
+
+    // Write 1 (only inserts)
+    String newCommitTime = "001";
+    String initCommitTime = "000";
+    int numRecords = 200;
+    castInsertFirstBatch(config, client, newCommitTime, initCommitTime, numRecords, BaseHoodieWriteClient::insert,
+        isPrepped, true, numRecords, populateMetaFields);
+
+    // Write 2 (updates)
+    String prevCommitTime = newCommitTime;
+    newCommitTime = "004";
+    numRecords = 100;
+    String commitTimeBetweenPrevAndNew = "002";
+    castUpdateBatch(config, client, newCommitTime, prevCommitTime,
+        Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
+        numRecords, 200, 2, populateMetaFields);
+
+    // Delete 1
+    prevCommitTime = newCommitTime;
+    newCommitTime = "005";
+    numRecords = 50;
+
+    castDeleteBatch(config, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
+        0, 150, config.populateMetaFields());
+
+    // Now simulate an upgrade and perform a restore operation
+    HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
+        TimelineLayoutVersion.CURR_VERSION).build();
+    client = getHoodieWriteClient(newConfig);
+
+    client.savepoint("004", "user1", "comment1");
+
+    client.restoreToInstant("004", config.isMetadataTableEnabled());
+
+    assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
+
+    client.deleteSavepoint("004");
+    assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
+
+    assertTheEntireDatasetHasAllRecordsStill(200);
+
+    // Perform Delete again on upgraded dataset.
+    prevCommitTime = newCommitTime;
+    newCommitTime = "006";
+    numRecords = 50;
+
+    castDeleteBatch(newConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true, 0, 150);
+
+    checkTimelineForUpsertsInternal(metaClient);
+
+    testMergeHandle(config);
+  }
+
+  /**
+   * Tests deletion of records.
+   */
+  protected void testDeletes(Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn,
+                             int numRecordsInSecondBatch, int expRecordsInSecondBatch, int expTotalRecords) throws Exception {
+    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
+    addConfigsForPopulateMetaFields(cfgBuilder, true);
+    BaseHoodieWriteClient client = getHoodieWriteClient(cfgBuilder.build());
+    /**
+     * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
+     */
+    String initCommitTime = "000";
+    String newCommitTime = "001";
+
+    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
+    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        (String instantTime, Integer numRecordsInThisCommit) -> {
+          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
+          List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
+
+          recordsInFirstBatch.addAll(fewRecordsForInsert);
+          recordsInFirstBatch.addAll(fewRecordsForDelete);
+          return recordsInFirstBatch;
+        };
+
+    castWriteBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
+        // unused as genFn uses hard-coded number of inserts/updates/deletes
+        -1, recordGenFunction, BaseHoodieWriteClient::upsert, true, 200, 200, 1, false, true);
+
+    /**
+     * Write 2 (deletes+writes).
+     */
+    String prevCommitTime = newCommitTime;
+    newCommitTime = "004";
+    recordGenFunction = secondBatchGenFn.apply(newCommitTime, numRecordsInSecondBatch, recordsInFirstBatch);
+
+    castWriteBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, numRecordsInSecondBatch, recordGenFunction,
+        BaseHoodieWriteClient::upsert, true, expRecordsInSecondBatch, expTotalRecords, 2, false, true);
+  }
+
+  protected void testRollbackFailedCommits(boolean populateMetaFields) throws Exception {
+    // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
+    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
+    HoodieTestUtils.init(storageConf, basePath);
+    BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+
+    // perform 1 successful commit
+    castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, true);
+
+    // Perform 2 failed writes to table
+    castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    // refresh data generator to delete records generated from failed commits
+    dataGen = new HoodieTestDataGenerator();
+    // Perform 1 successful write
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, true);
+    HoodieTableMetaClient metaClient = createMetaClient();
+
+    assertEquals(0, metaClient.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+    assertEquals(2, metaClient.getActiveTimeline().filterInflights().countInstants());
+    assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants());
+    // Wait till enough time passes such that the first 2 failed commits heartbeats are expired
+    boolean conditionMet = false;
+    while (!conditionMet) {
+      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
+      Thread.sleep(2000);
+    }
+    client.close();
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    // Perform 1 successful write
+    castWriteBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, true);
+    client.clean();
+    client.close();
+    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+    if (cleaningPolicy.isLazy()) {
+      assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+      // Since we write rollbacks not clean, there should be no clean action on the timeline
+      assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+      assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+    } else if (cleaningPolicy.isNever()) {
+      // never will get translated to Lazy if OCC is enabled.
+      assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+      // There should be no clean or rollback action on the timeline
+      assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+      assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+    }
+  }
+
+  protected void testRollbackFailedCommitsToggleCleaningPolicy(boolean populateMetaFields) throws Exception {
+    HoodieTestUtils.init(storageConf, basePath);
+    metaClient = createMetaClient();
+    HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
+    BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    // Perform 1 successful writes to table
+    castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, true);
+
+    // Perform 1 failed writes to table
+    castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    // Toggle cleaning policy to LAZY
+    cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
+    // Perform 2 failed writes to table
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    // Wait till enough time passes such that the 2 failed commits heartbeats are expired
+    boolean conditionMet = false;
+    while (!conditionMet) {
+      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
+      Thread.sleep(2000);
+    }
+    client.clean();
+    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+    assertEquals(3, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+    // Perform 2 failed commits
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 300,
+        0, false);
+    client.close();
+    // Toggle cleaning policy to EAGER
+    cleaningPolicy = EAGER;
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    client.startCommit();
+    client.close();
+    timeline = metaClient.getActiveTimeline().reload();
+    // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
+    assertEquals(3, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+    assertEquals(1, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+  }
+
+  protected void testParallelInsertAndCleanPreviousFailedCommits(boolean populateMetaFields) throws Exception {
+    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
+    ExecutorService service = Executors.newFixedThreadPool(2);
+    HoodieTestUtils.init(storageConf, basePath);
+    BaseHoodieWriteClient client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    // perform 1 successful write
+    castWriteBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+        0, true);
+
+    // Perform 2 failed writes to table
+    castWriteBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+        0, false);
+    client.close();
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    castWriteBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
+        100, dataGen::generateInserts, BaseHoodieWriteClient::bulkInsert, false, 100, 100,
+        0, false);
+    client.close();
+    // refresh data generator to delete records generated from failed commits
+    dataGen = new HoodieTestDataGenerator();
+    // Create a successful commit
+    Future<Object> commit3 = service.submit(() -> castWriteBatch(getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)),
+        "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
+        BaseHoodieWriteClient::bulkInsert, false, 100, 100, 0, true));
+    commit3.get();
+    HoodieTableMetaClient metaClient = createMetaClient();
+
+    assertEquals(0, metaClient.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+    assertEquals(2, metaClient.getActiveTimeline().filterInflights().countInstants());
+    assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants());
+    client = getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
+    // Wait till enough time passes such that the first 2 failed commits heartbeats are expired
+    boolean conditionMet = false;
+    while (!conditionMet) {
+      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
+      Thread.sleep(2000);
+    }
+    Future<Object> commit4 = service.submit(() -> castWriteBatch(getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)),
+        "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
+        BaseHoodieWriteClient::bulkInsert, false, 100, 100, 0, true));
+    Future<HoodieCleanMetadata> clean1 = service.submit(() -> getHoodieWriteClient(getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)).clean());
+    commit4.get();
+    clean1.get();
+    client.close();
+    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
+    assertEquals(2, timeline.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants());
+    // Since we write rollbacks not clean, there should be no clean action on the timeline
+    assertEquals(0, timeline.getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION)).countInstants());
+    assertEquals(3, timeline.getCommitsTimeline().filterCompletedInstants().countInstants());
+    service.shutdown();
   }
 }
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index 1e6e74664d8..39db790d795 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -1795,7 +1795,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
     tableType = HoodieTableType.COPY_ON_WRITE;
     init(tableType);
     context = new HoodieJavaEngineContext(storageConf);
-    HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+    HoodieWriteConfig config = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
     HoodieJavaWriteClient client = getHoodieWriteClient(config);
 
     // Write 1 (Bulk insert)
@@ -1869,7 +1869,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
     tableType = HoodieTableType.COPY_ON_WRITE;
     init(tableType);
     context = new HoodieJavaEngineContext(storageConf);
-    HoodieWriteConfig initialConfig = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+    HoodieWriteConfig initialConfig = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
     HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
         .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build()).build();
     HoodieJavaWriteClient client = getHoodieWriteClient(config);
@@ -2511,23 +2511,6 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
     doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
   }
 
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
-    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
-    return builder.withCompactionConfig(
-            HoodieCompactionConfig.newBuilder()
-                .compactionSmallFileSize(smallFileSize)
-                // Set rollback to LAZY so no inflights are deleted
-                .insertSplitSize(insertSplitSize).build())
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
-        .withStorageConfig(
-            HoodieStorageConfig.newBuilder()
-                .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
-                .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
-        .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
-        .build();
-  }
-
   public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, HoodieIndex.IndexType indexType,
                                                     HoodieFailedWritesCleaningPolicy cleaningPolicy) {
     return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schemaStr)
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index a6204ca75e6..3baf26257b8 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -18,88 +18,33 @@
 
 package org.apache.hudi.client.functional;
 
-import org.apache.hudi.avro.model.HoodieCleanMetadata;
-import org.apache.hudi.avro.model.HoodieClusteringPlan;
 import org.apache.hudi.client.BaseHoodieWriteClient;
 import org.apache.hudi.client.HoodieJavaWriteClient;
 import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy;
 import org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy;
 import org.apache.hudi.client.common.JavaTaskContextSupplier;
-import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
-import org.apache.hudi.common.config.HoodieStorageConfig;
-import org.apache.hudi.common.config.LockConfiguration;
-import org.apache.hudi.common.config.TypedProperties;
-import org.apache.hudi.common.data.HoodieData;
-import org.apache.hudi.common.data.HoodieListData;
 import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.ConsistencyGuardConfig;
 import org.apache.hudi.common.fs.FSUtils;
-import org.apache.hudi.common.model.HoodieAvroRecord;
 import org.apache.hudi.common.model.HoodieBaseFile;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
-import org.apache.hudi.common.model.HoodieFileGroupId;
-import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodieOperation;
-import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
 import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordDelegate;
 import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
-import org.apache.hudi.common.model.HoodieWriteStat;
-import org.apache.hudi.common.model.IOType;
-import org.apache.hudi.common.model.WriteConcurrencyMode;
-import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.marker.MarkerType;
-import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
-import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
 import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestTable;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.testutils.RawTripTestPayload;
-import org.apache.hudi.common.util.ClusteringUtils;
-import org.apache.hudi.common.util.CollectionUtils;
-import org.apache.hudi.common.util.MarkerUtils;
 import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.config.HoodieCleanConfig;
 import org.apache.hudi.config.HoodieClusteringConfig;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieLockConfig;
-import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieCommitException;
 import org.apache.hudi.exception.HoodieCorruptedDataException;
 import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.exception.HoodieUpsertException;
-import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.index.HoodieIndex.IndexType;
 import org.apache.hudi.io.HoodieMergeHandle;
 import org.apache.hudi.keygen.BaseKeyGenerator;
-import org.apache.hudi.keygen.KeyGenerator;
 import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.table.HoodieJavaTable;
 import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.HoodieWriteMetadata;
-import org.apache.hudi.table.action.commit.HoodieWriteHelper;
-import org.apache.hudi.table.marker.WriteMarkersFactory;
 import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
 
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -109,47 +54,18 @@ import org.junit.jupiter.params.provider.ValueSource;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
+import java.util.function.Function;
 import java.util.stream.Stream;
 
-import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
-import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
-import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
-import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
-import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
-import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 @SuppressWarnings("unchecked")
 public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTestHarness {
 
-  private static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
-
   private static Stream<Arguments> rollbackAfterConsistencyCheckFailureParams() {
     return Stream.of(
         Arguments.of(true),
@@ -157,19 +73,79 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
     );
   }
 
-  private HoodieTestTable testTable;
-
   @BeforeEach
   public void setUpTestTable() {
     testTable = HoodieMetadataTestTable.of(metaClient);
   }
 
+  private static final Function<Object, Object> IDENTITY = Function.identity();
+
+  private final Function<HoodieWriteConfig, BaseHoodieWriteClient> createBrokenClusteringClient =
+      config -> new WriteClientBrokenClustering<>(context, config);
+
+  private final Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTable =
+      (metaClient, config) -> getHoodieTable(metaClient, config);
+
+  @Override
+  protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                        String initCommitTime, int numRecordsInThisCommit,
+                                        Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                        boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+    return insertFirstBatch(writeConfig, (HoodieJavaWriteClient) client, newCommitTime, initCommitTime, numRecordsInThisCommit,
+        (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+        isPreppedAPI, assertForCommit, expRecordsInThisCommit, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                  Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                  Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+                                  Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+                                  boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+                                  boolean filterForCommitTimeWithAssert) throws Exception {
+    return writeBatch((HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+        (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+        assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                   Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                   Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+                                   boolean filterForCommitTimeWithAssert) throws Exception {
+    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates);
+
+    return writeBatch((HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+        (writeClient, records, commitTime) -> (List<WriteStatus>) writeFn.apply(writeClient, records, commitTime), assertForCommit, expRecordsInThisCommit, expTotalRecords,
+        expTotalCommits, false, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                   String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+    return deleteBatch(writeConfig, (HoodieJavaWriteClient) client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit,
+        isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
+    for (int i = 0; i < fullPartitionPaths.length; i++) {
+      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
+    }
+    assertEquals(200, countRowsInPaths(basePath, storage, fullPartitionPaths), "Must contain " + 200 + " records");
+    return fullPartitionPaths;
+  }
+
   /**
    * Test Auto Commit behavior for HoodieWriteClient insert API.
    */
   @Test
   public void testAutoCommitOnInsert() throws Exception {
-    testAutoCommit(HoodieJavaWriteClient::insert, false, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insert(recordRDD, instantTime), false, true);
   }
 
   /**
@@ -177,7 +153,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testAutoCommitOnInsertPrepped() throws Exception {
-    testAutoCommit(HoodieJavaWriteClient::insertPreppedRecords, true, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insertPreppedRecords(recordRDD, instantTime), true, true);
   }
 
   /**
@@ -185,7 +161,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testAutoCommitOnUpsert() throws Exception {
-    testAutoCommit(HoodieJavaWriteClient::upsert, false, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), false, true);
   }
 
   /**
@@ -193,7 +169,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testAutoCommitOnUpsertPrepped() throws Exception {
-    testAutoCommit(HoodieJavaWriteClient::upsertPreppedRecords, true, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
   }
 
   /**
@@ -201,7 +177,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testAutoCommitOnBulkInsert() throws Exception {
-    testAutoCommit(HoodieJavaWriteClient::bulkInsert, false, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsert(recordRDD, instantTime), false, true);
   }
 
   /**
@@ -213,48 +189,12 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
         Option.empty()), true, true);
   }
 
-  /**
-   * Test auto-commit by applying the write function.
-   *
-   * @param writeFn One of HoodieWriteClient Write API
-   * @throws Exception in case of failure
-   */
-  private void testAutoCommit(Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn,
-                              boolean isPrepped, boolean populateMetaFields) throws Exception {
-    // Set autoCommit false
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());) {
-
-      String prevCommitTime = "000";
-      String newCommitTime = "001";
-      int numRecords = 200;
-      List<WriteStatus> result = insertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn,
-          isPrepped, false, numRecords);
-
-      assertFalse(testTable.commitExists(newCommitTime),
-          "If Autocommit is false, then commit should not be made automatically");
-      assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
-      assertTrue(testTable.commitExists(newCommitTime),
-          "After explicit commit, commit file should be created");
-    }
-  }
-
-  private void insertWithConfig(HoodieWriteConfig config, int numRecords, String instant) throws Exception {
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(config)) {
-      Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
-          writeClient.bulkInsert(recordRDD, instantTime, Option.empty());
-      List<WriteStatus> result = insertFirstBatch(config, client, instant,
-          "000", numRecords, writeFn, false, false, numRecords);
-    }
-  }
-
   /**
    * Test De-duplication behavior for HoodieWriteClient insert API.
    */
   @Test
   public void testDeduplicationOnInsert() throws Exception {
-    testDeduplication(HoodieJavaWriteClient::insert, true);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.insert(records, commitTime), true, false);
   }
 
   /**
@@ -262,7 +202,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testDeduplicationKeepOperationFieldOnInsert() throws Exception {
-    testDeduplicationKeepOperation(HoodieJavaWriteClient::insert, true);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.insert(records, commitTime), true, true);
   }
 
   /**
@@ -270,7 +210,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testDeduplicationOnBulkInsert() throws Exception {
-    testDeduplication(HoodieJavaWriteClient::bulkInsert, true);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.bulkInsert(records, commitTime), true, false);
   }
 
   /**
@@ -278,143 +218,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testDeduplicationOnUpsert() throws Exception {
-    testDeduplication(HoodieJavaWriteClient::upsert, true);
-  }
-
-  /**
-   * Test Deduplication Logic for write function.
-   *
-   * @param writeFn One of HoodieWriteClient non-prepped write APIs
-   * @throws Exception in case of failure
-   */
-  private void testDeduplication(
-      Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
-    String newCommitTime = "001";
-
-    String recordKey = UUID.randomUUID().toString();
-    HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
-    HoodieRecord<RawTripTestPayload> recordOne =
-        new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime));
-
-    HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
-    HoodieRecord recordTwo =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
-    // Same key and partition as keyTwo
-    HoodieRecord recordThree =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
-    HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieListData.eager(Arrays.asList(recordOne, recordTwo, recordThree));
-    HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
-        .combineInput(true, true);
-    addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
-    HoodieWriteConfig writeConfig = configBuilder.build();
-
-    // Global dedup should be done based on recordKey only
-    HoodieIndex index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(true);
-    int dedupParallelism = records.getNumPartitions() + 100;
-    HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(records.getNumPartitions(), dedupedRecsRdd.getNumPartitions());
-    assertEquals(1, dedupedRecs.size());
-    assertEquals(dedupedRecs.get(0).getPartitionPath(), recordThree.getPartitionPath());
-    assertNodupesWithinPartition(dedupedRecs);
-
-    // non-Global dedup should be done based on both recordKey and partitionPath
-    index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(false);
-    dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(records.getNumPartitions(), dedupedRecsRdd.getNumPartitions());
-    assertEquals(2, dedupedRecs.size());
-    assertNodupesWithinPartition(dedupedRecs);
-
-    // Perform write-action and check
-    List<HoodieRecord> recordList = Arrays.asList(recordOne, recordTwo, recordThree);
-
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(writeConfig)) {
-      client.startCommitWithTime(newCommitTime);
-      List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
-      assertNoWriteErrors(statuses);
-      assertEquals(2, statuses.size());
-      assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
-          .collect(Collectors.toList()));
-    }
-  }
-
-  /**
-   * Test Deduplication Logic for write function.
-   *
-   * @param writeFn One of HoodieWriteClient non-prepped write APIs
-   * @throws Exception in case of failure
-   */
-  private void testDeduplicationKeepOperation(
-      Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
-    String newCommitTime = "001";
-
-    String recordKey = UUID.randomUUID().toString();
-    HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
-    HoodieRecord<RawTripTestPayload> recordOne =
-        new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime), HoodieOperation.INSERT);
-
-    HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
-    HoodieRecord recordTwo =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.INSERT);
-
-    // Same key and partition as keyTwo
-    HoodieRecord recordThree =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.UPDATE_AFTER);
-
-    HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieListData.eager(Arrays.asList(recordOne, recordTwo, recordThree));
-    HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAllowOperationMetadataField(true)
-        .combineInput(true, true);
-    addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
-    HoodieWriteConfig writeConfig = configBuilder.build();
-
-    // Global dedup should be done based on recordKey only
-    HoodieIndex index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(true);
-    int dedupParallelism = records.getNumPartitions() + 100;
-    HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(dedupedRecs.get(0).getOperation(), recordThree.getOperation());
-
-    // Perform write-action and check
-    List<HoodieRecord> recordList = Arrays.asList(recordOne, recordTwo, recordThree);
-
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(writeConfig)) {
-      client.startCommitWithTime(newCommitTime);
-      List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime);
-      assertNoWriteErrors(statuses);
-      assertEquals(2, statuses.size());
-      assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
-          .collect(Collectors.toList()));
-    }
-  }
-
-  /**
-   * Assert that there is no duplicate key at the partition level.
-   *
-   * @param recordDelegates List of Hoodie record delegates
-   */
-  void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
-    Map<String, Set<String>> partitionToKeys = new HashMap<>();
-    for (HoodieRecordDelegate r : recordDelegates) {
-      String recordKey = r.getRecordKey();
-      String partitionPath = r.getPartitionPath();
-      if (!partitionToKeys.containsKey(partitionPath)) {
-        partitionToKeys.put(partitionPath, new HashSet<>());
-      }
-      assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
-      partitionToKeys.get(partitionPath).add(recordKey);
-    }
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) client.upsert(records, commitTime), true, false);
   }
 
   /**
@@ -422,9 +226,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testUpserts() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsert, false);
+    testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), true, false);
   }
 
   /**
@@ -432,134 +234,23 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testUpsertsPrepped() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsertPreppedRecords, true);
+    testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
   }
 
-  /**
-   * Test one of HoodieWriteClient upsert(Prepped) APIs.
-   *
-   * @param config  Write Config
-   * @param writeFn One of Hoodie Write Function API
-   * @throws Exception in case of error
-   */
-  private void testUpsertsInternal(HoodieWriteConfig config,
-                                   Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPrepped)
-      throws Exception {
-    // Force using older timeline layout
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
-        .withRollbackUsingMarkers(true)
-        .withProps(config.getProps()).withTimelineLayoutVersion(
-            VERSION_0).build();
-
-    HoodieTableMetaClient.withPropertyBuilder()
-        .fromMetaClient(metaClient)
-        .setTimelineLayoutVersion(VERSION_0)
-        .setPopulateMetaFields(config.populateMetaFields())
-        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
-    HoodieJavaWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String newCommitTime = "001";
-    String initCommitTime = "000";
-    int numRecords = 200;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, HoodieJavaWriteClient::insert,
-        isPrepped, true, numRecords, config.populateMetaFields());
-
-    // Write 2 (updates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    numRecords = 100;
-    String commitTimeBetweenPrevAndNew = "002";
-    updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime,
-        Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
-        numRecords, 200, 2, config.populateMetaFields());
-
-    // Delete 1
-    prevCommitTime = newCommitTime;
-    newCommitTime = "005";
-    numRecords = 50;
-
-    deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
-        0, 150, config.populateMetaFields());
-
-    // Now simulate an upgrade and perform a restore operation
-    HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
-        TimelineLayoutVersion.CURR_VERSION).build();
-    client = getHoodieWriteClient(newConfig);
-
-    client.savepoint("004", "user1", "comment1");
-
-    client.restoreToInstant("004", config.isMetadataTableEnabled());
-
-    assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
-
-    client.deleteSavepoint("004");
-    assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
-
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(200, countRowsInPaths(basePath, storage, fullPartitionPaths),
-        "Must contain " + 200 + " records");
-
-    // Perform Delete again on upgraded dataset.
-    prevCommitTime = newCommitTime;
-    newCommitTime = "006";
-    numRecords = 50;
-
-    deleteBatch(newConfig, client, newCommitTime, prevCommitTime,
-        initCommitTime, numRecords, isPrepped, true,
-        0, 150);
-
-    HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
-    List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
-    assertEquals(5, instants.size());
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"),
-        instants.get(0));
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"),
-        instants.get(1));
-    // New Format should have all states of instants
-    assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"),
-        instants.get(2));
-    assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"),
-        instants.get(3));
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"),
-        instants.get(4));
-
-    final HoodieWriteConfig cfg = hoodieWriteConfig;
+  @Override
+  protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
     final String instantTime = "007";
     HoodieTableMetaClient metaClient = createMetaClient();
-    String basePathStr = basePath;
-    HoodieTable table = getHoodieTable(metaClient, cfg);
-    String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
-    HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
-        .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
-                metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
-            HoodieCommitMetadata.class);
-    String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
-        .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
-        .map(ee -> ee.getPath()).orElse(null);
-    String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
-        .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
-        .map(ee -> ee.getPartitionPath()).orElse(null);
-    Path baseFilePath = new Path(basePathStr, filePath);
-    HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath.toString());
+    HoodieTable table = getHoodieTable(metaClient, config);
+    Pair<String, String> partitionAndBaseFilePaths = getPartitionAndBaseFilePathsFromLatestCommitMetadata(metaClient);
+    HoodieBaseFile baseFile = new HoodieBaseFile(partitionAndBaseFilePaths.getRight());
 
     HoodieMergeHandle handle = null;
     try {
-      handle = new HoodieMergeHandle(cfg, instantTime, table, new HashMap<>(),
-          partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new JavaTaskContextSupplier(),
+      handle = new HoodieMergeHandle(config, instantTime, table, new HashMap<>(),
+          partitionAndBaseFilePaths.getLeft(), FSUtils.getFileId(baseFile.getFileName()), baseFile, new JavaTaskContextSupplier(),
           config.populateMetaFields() ? Option.empty() :
-              Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
-      WriteStatus writeStatus = new WriteStatus(false, 0.0);
-      writeStatus.setStat(new HoodieWriteStat());
-      writeStatus.getStat().setNumWrites(0);
-      handle.performMergeDataValidationCheck(writeStatus);
+              Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(config.getProps())));
     } catch (HoodieCorruptedDataException e1) {
       fail("Exception not expected because merge validation check is disabled");
     } finally {
@@ -571,13 +262,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
     handle = null;
     try {
       final String newInstantTime = "006";
-      cfg.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
-      HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(cfg.getProps()).build();
+      config.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
+      HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(config.getProps()).build();
       // does the handle need to be closed to clean up the writer it contains?
       handle = new HoodieMergeHandle(cfg2, newInstantTime, table, new HashMap<>(),
-          partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new JavaTaskContextSupplier(),
+          partitionAndBaseFilePaths.getLeft(), FSUtils.getFileId(baseFile.getFileName()), baseFile, new JavaTaskContextSupplier(),
           config.populateMetaFields() ? Option.empty() :
-              Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+              Option.of((BaseKeyGenerator) HoodieAvroKeyGeneratorFactory.createKeyGenerator(config.getProps())));
       fail("The above line should have thrown an exception");
     } catch (HoodieUpsertException e2) {
       // expected
@@ -593,58 +284,15 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testInsertsWithHoodieConcatHandle() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testHoodieConcatHandle(cfgBuilder.build(), false);
+    testHoodieConcatHandle(true, false);
   }
 
   /**
    * Test InsertPrepped API for HoodieConcatHandle.
    */
+  @Test
   public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testHoodieConcatHandle(cfgBuilder.build(), true);
-  }
-
-  /**
-   * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
-   *
-   * @param config Write Config
-   * @throws Exception in case of error
-   */
-  private void testHoodieConcatHandle(HoodieWriteConfig config, boolean isPrepped)
-      throws Exception {
-    // Force using older timeline layout
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
-        .withProps(config.getProps()).withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(
-            VERSION_0).build();
-    HoodieTableMetaClient.withPropertyBuilder()
-        .fromMetaClient(metaClient)
-        .setTimelineLayoutVersion(VERSION_0)
-        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
-    HoodieJavaWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String newCommitTime = "001";
-    String initCommitTime = "000";
-    int numRecords = 200;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, HoodieJavaWriteClient::insert,
-        isPrepped, true, numRecords, config.populateMetaFields());
-
-    // Write 2 (updates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    numRecords = 100;
-    String commitTimeBetweenPrevAndNew = "002";
-
-    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
-
-    writeBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime,
-        numRecords, recordGenFunction, HoodieJavaWriteClient::insert, true, numRecords, 300,
-        2, false, config.populateMetaFields());
+    testHoodieConcatHandle(true, true);
   }
 
   /**
@@ -652,8 +300,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testInsertsWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), false);
+    testHoodieConcatHandleOnDupInserts(false);
   }
 
   /**
@@ -661,83 +308,25 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testInsertsPreppedWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), true);
-  }
-
-  private void testHoodieConcatHandleOnDupInserts(HoodieWriteConfig config, boolean isPrepped) throws Exception {
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
-        .withProps(config.getProps())
-        .withMergeAllowDuplicateOnInserts(true)
-        .build();
-
-    HoodieJavaWriteClient<RawTripTestPayload> client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-    int firstInsertRecords = 50;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, firstInsertRecords, HoodieJavaWriteClient::insert,
-        isPrepped, true, firstInsertRecords, config.populateMetaFields());
-
-    // Write 2 (updates with duplicates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
-    List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
-
-    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUpdates);
-
-    writeBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
-        secondInsertRecords, recordGenFunction, HoodieJavaWriteClient::insert, true, secondInsertRecords,
-        firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+    testHoodieConcatHandleOnDupInserts(true);
   }
 
   /**
    * Tests deletion of records.
    */
+  @Test
   public void testDeletes() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());
-    /**
-     * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
-     */
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-
-    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
-    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        (String instantTime, Integer numRecordsInThisCommit) -> {
-          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
-          List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
-
-          recordsInFirstBatch.addAll(fewRecordsForInsert);
-          recordsInFirstBatch.addAll(fewRecordsForDelete);
-          return recordsInFirstBatch;
+    Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn =
+        (String instantTime, Integer numRecordsInThisCommit, List<HoodieRecord> recordsInFirstBatch) -> {
+          List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
+          return (time, numRecords) -> {
+            List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
+            recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+            recordsInSecondBatch.addAll(dataGen.generateInserts(instantTime, 25));
+            return recordsInSecondBatch;
+          };
         };
-    writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
-        // unused as genFn uses hard-coded number of inserts/updates/deletes
-        -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 200, 200, 1, false,
-        true);
-
-    /**
-     * Write 2 (deletes+writes).
-     */
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    final List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
-
-    recordGenFunction = (String instantTime, Integer numRecordsInThisCommit) -> {
-      List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
-      recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
-      recordsInSecondBatch.addAll(dataGen.generateInserts(instantTime, 25));
-      return recordsInSecondBatch;
-    };
-    writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 75, recordGenFunction,
-        HoodieJavaWriteClient::upsert, true, 25, 175, 2, false,
-        true);
+    super.testDeletes(secondBatchGenFn, 75, 25, 175);
   }
 
   /**
@@ -748,367 +337,41 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testDeletesForInsertsInSameBatch() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());
-    /**
-     * Write 200 inserts and issue deletes to a subset(50) of inserts.
-     */
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-
-    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
-    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        (String instantTime, Integer numRecordsInThisCommit) -> {
-          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
-          List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
-
-          recordsInFirstBatch.addAll(fewRecordsForInsert);
-          recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
-          return recordsInFirstBatch;
-        };
-
-    writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
-        -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 150, 150, 1, false,
-        true);
+    super.testDeletesForInsertsInSameBatch();
   }
 
   @Test
   public void testAndValidateClusteringOutputFiles() throws IOException {
-    String partitionPath = "2015/03/16";
-    testInsertTwoBatches(true, partitionPath);
-
-    // Trigger clustering
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
-        .withClusteringConfig(HoodieClusteringConfig.newBuilder()
-            .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
-            .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
-            .withInlineClustering(true).withInlineClusteringNumCommits(2).build());
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
-      int numRecords = 200;
-      String newCommitTime = client.createNewInstantTime();
-      List<HoodieRecord> records1 = dataGen.generateInserts(newCommitTime, numRecords);
-      client.startCommitWithTime(newCommitTime);
-      List<WriteStatus> statuses = client.insert(records1, newCommitTime);
-      client.commit(newCommitTime, statuses);
-      assertNoWriteErrors(statuses);
-
-      metaClient = HoodieTableMetaClient.reload(metaClient);
-      HoodieInstant replaceCommitInstant =
-          metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
-      HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
-          .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
-              HoodieReplaceCommitMetadata.class);
-
-      List<String> filesFromReplaceCommit = new ArrayList<>();
-      replaceCommitMetadata.getPartitionToWriteStats()
-          .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
-
-      // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
-      List<StoragePathInfo> pathInfoList =
-          storage.listDirectEntries(new StoragePath(basePath + "/" + partitionPath));
-      List<String> clusteredFiles = pathInfoList.stream()
-          .filter(
-              entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
-          .map(fileStatus -> partitionPath + "/" + fileStatus.getPath().getName())
-          .collect(Collectors.toList());
-      assertEquals(clusteredFiles, filesFromReplaceCommit);
-    }
+    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder()
+        .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
+        .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
+        .withInlineClustering(true).withInlineClusteringNumCommits(2).build();
+    testAndValidateClusteringOutputFiles(createBrokenClusteringClient, clusteringConfig, IDENTITY, IDENTITY);
   }
 
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testInlineScheduleClustering(boolean scheduleInlineClustering) throws IOException {
-    testInsertTwoBatches(true);
-
-    // setup clustering config.
     HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
         .withClusteringTargetPartitions(0).withAsyncClusteringMaxCommits(1).withInlineClustering(false).withScheduleInlineClustering(scheduleInlineClustering)
         .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
-        .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName())
-        .build();
-
-    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
-        .withClusteringConfig(clusteringConfig)
-        .withProps(getPropertiesForKeyGen()).build();
-    HoodieJavaWriteClient client = getHoodieWriteClient(config);
-    dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
-    String commitTime1 = client.createNewInstantTime();
-    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
-    client.startCommitWithTime(commitTime1);
-    List<WriteStatus> statuses = client.insert(records1, commitTime1);
-    assertNoWriteErrors(statuses);
-    client.commit(commitTime1, statuses);
-
-    HoodieTableMetaClient metaClient = createMetaClient();
-    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
-        ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
-    if (scheduleInlineClustering) {
-      assertEquals(1, pendingClusteringPlans.size());
-    } else {
-      assertEquals(0, pendingClusteringPlans.size());
-    }
-  }
-
-  private List<HoodieRecord> testInsertAndClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields,
-                                                     boolean completeClustering, boolean assertSameFileIds, String validatorClasses,
-                                                     String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation) throws Exception {
-    Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields);
-    testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords);
-    return allRecords.getLeft().getLeft();
+        .withClusteringPlanStrategyClass(JavaSizeBasedClusteringPlanStrategy.class.getName()).build();
+    testInlineScheduleClustering(createBrokenClusteringClient, clusteringConfig, IDENTITY, IDENTITY);
   }
 
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail) throws IOException {
-    try {
-      Properties properties = new Properties();
-      properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
-      properties.setProperty("hoodie.auto.commit", "false");
-      properties.setProperty("hoodie.clustering.inline.max.commits", "1");
-      properties.setProperty("hoodie.clustering.inline", "true");
-      testInsertTwoBatches(true, "2015/03/16", properties, true);
-      assertFalse(shouldFail);
-    } catch (HoodieException e) {
-      assertEquals(CLUSTERING_FAILURE, e.getMessage());
-      assertTrue(shouldFail);
-    }
-  }
-
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields) throws IOException {
-    return testInsertTwoBatches(populateMetaFields, "2015/03/16");
-  }
-
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath) throws IOException {
-    return testInsertTwoBatches(populateMetaFields, partitionPath, new Properties(), false);
-  }
-
-  /**
-   * This method returns following three items:
-   * 1. List of all HoodieRecord written in the two batches of insert.
-   * 2. Commit instants of the two batches.
-   * 3. List of new file group ids that were written in the two batches.
-   */
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props,
-                                                                                                    boolean failInlineClustering) throws IOException {
-    // create config to not update small files.
-    HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
-        populateMetaFields ? props : getPropertiesForKeyGen());
-    HoodieJavaWriteClient client;
-    if (failInlineClustering) {
-      if (null != writeClient) {
-        writeClient.close();
-        writeClient = null;
-      }
-      client = new WriteClientBrokenClustering(context, config);
-    } else {
-      client = getHoodieWriteClient(config);
-    }
-
-    dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
-    String commitTime1 = client.createNewInstantTime();
-    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
-    List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
-    Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
-
-    String commitTime2 = client.createNewInstantTime();
-    List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
-    List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
-    client.close();
-    Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
-    Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
-    fileIdsUnion.addAll(fileIds2);
-    //verify new files are created for 2nd write
-    Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
-    fileIdIntersection.retainAll(fileIds2);
-    assertEquals(0, fileIdIntersection.size());
-    return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
-  }
-
-  private void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
-                              String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
-                              Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords) throws IOException {
-
-    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
-        .withClusteringConfig(clusteringConfig)
-        .withProps(getPropertiesForKeyGen()).build();
-    HoodieWriteMetadata<List<WriteStatus>> clusterMetadata =
-        performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords.getLeft());
-    if (assertSameFileIds) {
-      Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
-          .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
-      Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
-      assertEquals(insertedFileIds, replacedFileIds);
-    }
-    if (completeClustering) {
-      String clusteringCommitTime = metaClient.reloadActiveTimeline().getCompletedReplaceTimeline()
-          .getReverseOrderedInstants().findFirst().get().getTimestamp();
-      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(), clusterMetadata.getWriteStatuses(), config);
-    }
-  }
-
-  private HoodieWriteMetadata<List<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
-                                                                   boolean populateMetaFields,
-                                                                   boolean completeClustering,
-                                                                   String validatorClasses,
-                                                                   String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
-                                                                   Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException {
-    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
-        .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
-        .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
-        .build();
-
-    HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
-        .withPreCommitValidatorConfig(validatorConfig)
-        .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
-        .withClusteringConfig(clusteringConfig).build();
-
-    // create client with new config.
-    HoodieJavaWriteClient client = getHoodieWriteClient(config);
-    String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
-    HoodieWriteMetadata<List<WriteStatus>> clusterMetadata = client.cluster(clusteringCommitTime, completeClustering);
-    if (config.populateMetaFields()) {
-      verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses());
-    } else {
-      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses(), config);
-    }
-
-    Set<HoodieFileGroupId> replacedFileIds = new HashSet<>();
-    clusterMetadata.getPartitionToReplaceFileIds().entrySet().forEach(partitionFiles ->
-        partitionFiles.getValue().stream().forEach(file ->
-            replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file))));
-    return clusterMetadata;
-  }
-
-  private Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
-    return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
-  }
-
-  private Set<String> insertPartitionRecordsWithCommit(HoodieJavaWriteClient client, int recordsCount, String commitTime1, String partitionPath) throws IOException {
-    client.startCommitWithTime(commitTime1);
-    List<HoodieRecord> inserts1 = dataGen.generateInsertsForPartition(commitTime1, recordsCount, partitionPath);
-    List<WriteStatus> statuses = client.upsert(inserts1, commitTime1);
-    assertNoWriteErrors(statuses);
-    Set<String> batchBuckets = statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
-    verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig());
-    return batchBuckets;
-  }
-
-  /**
-   * Verify data in base files matches expected records and commit time.
-   */
-  private void verifyRecordsWritten(String commitTime, boolean populateMetadataField,
-                                    List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, HoodieWriteConfig config) throws IOException {
-    List<GenericRecord> records = new ArrayList<>();
-    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
-    if (config.populateMetaFields()) {
-      for (GenericRecord record : records) {
-        String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
-        assertEquals(commitTime,
-            record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
-        assertTrue(expectedKeys.contains(recordKey));
-      }
-    } else {
-      KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()));
-      for (GenericRecord record : records) {
-        String recordKey = keyGenerator.getKey(record).getRecordKey();
-        if (!populateMetadataField) {
-          assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
-        }
-        assertTrue(expectedKeys.contains(recordKey));
-      }
-    }
-  }
-
-  @NotNull
-  private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
-    for (WriteStatus status : allStatus) {
-      StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
-      records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
-    }
-    Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
-    assertEquals(records.size(), expectedKeys.size());
-    return expectedKeys;
-  }
-
-  private void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
-    List<GenericRecord> records = new ArrayList<>();
-    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
-    Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
-        .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
-    assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
-    Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
-    for (GenericRecord record : records) {
-      String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
-      assertTrue(expectedKeys.contains(recordKey));
-      String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
-      assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
-    }
-  }
-
-  private List<WriteStatus> writeAndVerifyBatch(HoodieJavaWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
-    return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
-  }
-
-  private List<WriteStatus> writeAndVerifyBatch(HoodieJavaWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
-    client.startCommitWithTime(commitTime);
-    List<WriteStatus> statusRDD = client.upsert(inserts, commitTime);
-    if (autoCommitOff) {
-      client.commit(commitTime, statusRDD);
-    }
-    List<WriteStatus> statuses = statusRDD;
-    assertNoWriteErrors(statuses);
-    verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig());
-
-    return statuses;
-  }
-
-  private Pair<Set<String>, List<HoodieRecord>> testUpdates(String instantTime, HoodieJavaWriteClient client,
-                                                            int sizeToInsertAndUpdate, int expectedTotalRecords)
-      throws IOException {
-    client.startCommitWithTime(instantTime);
-    List<HoodieRecord> inserts = dataGen.generateInserts(instantTime, sizeToInsertAndUpdate);
-    Set<String> keys = recordsToRecordKeySet(inserts);
-    List<HoodieRecord> insertsAndUpdates = new ArrayList<>();
-    insertsAndUpdates.addAll(inserts);
-    insertsAndUpdates.addAll(dataGen.generateUpdates(instantTime, inserts));
-
-    List<WriteStatus> statuses = client.upsert(insertsAndUpdates, instantTime);
-    assertNoWriteErrors(statuses);
-
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(expectedTotalRecords,
-        countRowsInPaths(basePath, storage, fullPartitionPaths),
-        "Must contain " + expectedTotalRecords + " records");
-    return Pair.of(keys, inserts);
+    testFailWritesOnInlineTableServiceExceptions(shouldFail, createBrokenClusteringClient);
   }
 
   /**
    * Test delete with delete api.
    */
-  @Test
-  public void testDeletesWithoutInserts() {
-    final String testPartitionPath = "2016/09/26";
-    final int insertSplitLimit = 100;
-    // setup the small file handling params
-    HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
-        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), true, true
-            ? new Properties() : getPropertiesForKeyGen());
-    dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
-    HoodieJavaWriteClient client = getHoodieWriteClient(config);
-
-    // delete non existent keys
-    String commitTime1 = "001";
-    client.startCommitWithTime(commitTime1);
-
-    List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
-    List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
-    client.delete(hoodieKeysToDelete, commitTime1);
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testDeletesWithoutInserts(boolean populateMetaFields) {
+    testDeletesWithoutInserts(populateMetaFields, IDENTITY, IDENTITY);
   }
 
   /**
@@ -1116,104 +379,16 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
    */
   @Test
   public void testCommitWritesRelativePaths() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
-      HoodieTableMetaClient metaClient = createMetaClient();
-      HoodieJavaTable table = HoodieJavaTable.create(cfgBuilder.build(), context, metaClient);
-
-      String instantTime = "000";
-      client.startCommitWithTime(instantTime);
-
-      List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
-
-      List<WriteStatus> result = client.bulkInsert(records, instantTime);
-
-      assertTrue(client.commit(instantTime, result), "Commit should succeed");
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-
-      // Get base file paths from commit metadata
-      String actionType = metaClient.getCommitActionType();
-      HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
-      HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
-      HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
-          .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(),
-              HoodieCommitMetadata.class);
-      String basePath = table.getMetaClient().getBasePath();
-      Collection<String> commitPathNames =
-          commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
-
-      // Read from commit file
-      HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
-          metaClient.reloadActiveTimeline()
-              .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
-          HoodieCommitMetadata.class);
-      HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
-      // Compare values in both to make sure they are equal.
-      for (String pathName : paths.values()) {
-        assertTrue(commitPathNames.contains(pathName));
-      }
-    }
+    testCommitWritesRelativePaths(IDENTITY);
   }
 
   /**
    * Test to ensure commit metadata points to valid files.10.
    */
-  @Test
-  public void testMetadataStatsOnCommit() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    HoodieWriteConfig cfg = cfgBuilder.build();
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-
-    String instantTime0 = "000";
-    client.startCommitWithTime(instantTime0);
-
-    List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
-    List<WriteStatus> result0 = client.bulkInsert(records0, instantTime0);
-
-    assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
-    assertTrue(testTable.commitExists(instantTime0),
-        "After explicit commit, commit file should be created");
-
-    // Read from commit file
-    HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
-        metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
-        HoodieCommitMetadata.class);
-    int inserts = 0;
-    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
-      for (HoodieWriteStat stat : pstat.getValue()) {
-        inserts += stat.getNumInserts();
-      }
-    }
-    assertEquals(200, inserts);
-
-    // Update + Inserts such that they just expand file1
-    String instantTime1 = "001";
-    client.startCommitWithTime(instantTime1);
-
-    List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
-    List<WriteStatus> result1 = client.upsert(records1, instantTime1);
-
-    assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
-    assertTrue(testTable.commitExists(instantTime1),
-        "After explicit commit, commit file should be created");
-
-    // Read from commit file
-    metadata = HoodieCommitMetadata.fromBytes(
-        metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
-        HoodieCommitMetadata.class);
-    inserts = 0;
-    int upserts = 0;
-    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
-      for (HoodieWriteStat stat : pstat.getValue()) {
-        inserts += stat.getNumInserts();
-        upserts += stat.getNumUpdateWrites();
-      }
-    }
-    assertEquals(0, inserts);
-    assertEquals(200, upserts);
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testMetadataStatsOnCommit(boolean populateMetaFields) throws Exception {
+    testMetadataStatsOnCommit(populateMetaFields, IDENTITY);
   }
 
   /**
@@ -1222,71 +397,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
-    HoodieTableMetaClient metaClient = createMetaClient();
-    String instantTime = "000";
-    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
-        .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-    Pair<StoragePath, List<WriteStatus>> result = testConsistencyCheck(
-        metaClient, instantTime, enableOptimisticConsistencyGuard);
-
-    // Delete orphan marker and commit should succeed
-    metaClient.getStorage().deleteFile(result.getKey());
-    if (!enableOptimisticConsistencyGuard) {
-      assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-      // Marker directory must be removed
-      assertFalse(metaClient.getStorage()
-          .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    } else {
-      // with optimistic, first client.commit should have succeeded.
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-      // Marker directory must be removed
-      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    }
+    testConsistencyCheckDuringFinalize(context, enableOptimisticConsistencyGuard, getHoodieTable, IDENTITY, IDENTITY);
   }
 
   private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
                                                                      boolean populateMetaFields) throws Exception {
-    String instantTime = "00000000000010";
-    HoodieTableMetaClient metaClient = createMetaClient();
-
-    Properties properties = new Properties();
-    if (!populateMetaFields) {
-      properties = getPropertiesForKeyGen();
-    }
-
-    HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build() :
-        getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
-            .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
-                .withConsistencyCheckEnabled(true)
-                .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
-                .withOptimisticConsistencyGuardSleepTimeMs(1).build())
-            .withProperties(properties).build();
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-    testConsistencyCheck(metaClient, instantTime, enableOptimisticConsistencyGuard);
-
-    if (!enableOptimisticConsistencyGuard) {
-      // Rollback of this commit should succeed with FailSafeCG
-      client.rollback(instantTime);
-      assertFalse(testTable.commitExists(instantTime),
-          "After explicit rollback, commit file should not be present");
-      // Marker directory must be removed after rollback
-      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    } else {
-      // if optimistic CG is enabled, commit should have succeeded.
-      assertTrue(testTable.commitExists(instantTime),
-          "With optimistic CG, first commit should succeed. commit file should be present");
-      // Marker directory must be removed after rollback
-      assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-      client.rollback(instantTime);
-      assertFalse(testTable.commitExists(instantTime),
-          "After explicit rollback, commit file should not be present");
-    }
+    testRollbackAfterConsistencyCheckFailureUsingFileList(context, rollbackUsingMarkers, enableOptimisticConsistencyGuard,
+        populateMetaFields, getHoodieTable, IDENTITY, IDENTITY);
   }
 
   @ParameterizedTest
@@ -1303,303 +420,17 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
 
   @Test
   public void testRollbackFailedCommits() throws Exception {
-    // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
-    boolean populateMetaFields = true;
-    HoodieTestUtils.init(storageConf, basePath);
-    HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-
-    // perform 1 successful commit
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-
-    // Perform 2 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // refresh data generator to delete records generated from failed commits
-    dataGen = new HoodieTestDataGenerator();
-    // Perform 1 successful write
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-    HoodieTableMetaClient metaClient = createMetaClient();
-
-    assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
-    assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
-    assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
-    // Await till enough time passes such that the first 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
-      Thread.sleep(2000);
-    }
-    client.close();
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    // Perform 1 successful write
-    writeBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-    client.clean();
-    client.close();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    if (cleaningPolicy.isLazy()) {
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
-              .countInstants()
-              == 2);
-      // Since we write rollbacks not clean, there should be no clean action on the timeline
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
-              .countInstants()
-              == 0);
-      assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    } else if (cleaningPolicy.isNever()) {
-      // never will get translated to Lazy if OCC is enabled.
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
-              .countInstants()
-              == 2);
-      // There should be no clean or rollback action on the timeline
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
-              .countInstants()
-              == 0);
-      assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    }
+    super.testRollbackFailedCommits(true);
   }
 
   @Test
   public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception {
-    HoodieTestUtils.init(storageConf, basePath);
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
-    HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    // Perform 1 successful writes to table
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-
-    // Perform 1 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Toggle cleaning policy to LAZY
-    cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
-    // Perform 2 failed writes to table
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Await till enough time passes such that the 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
-      Thread.sleep(2000);
-    }
-    client.clean();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
-    // Perform 2 failed commits
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Toggle cleaning policy to EAGER
-    cleaningPolicy = EAGER;
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    client.startCommit();
-    timeline = metaClient.getActiveTimeline().reload();
-    // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
-    assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1);
-    client.close();
+    super.testRollbackFailedCommitsToggleCleaningPolicy(true);
   }
 
   @Test
   public void testParallelInsertAndCleanPreviousFailedCommits() throws Exception {
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
-    ExecutorService service = Executors.newFixedThreadPool(2);
-    HoodieTestUtils.init(storageConf, basePath);
-    HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    // perform 1 successful write
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
-        0, true);
-
-    // Perform 2 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
-        0, false);
-    client.close();
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 100,
-        0, false);
-    client.close();
-    // refresh data generator to delete records generated from failed commits
-    dataGen = new HoodieTestDataGenerator();
-    // Create a successful commit
-    Future<List<WriteStatus>> commit3 = service.submit(() -> writeBatch(new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
-        "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
-        HoodieJavaWriteClient::bulkInsert, false, 100, 100, 0, true));
-    commit3.get();
-    HoodieTableMetaClient metaClient = createMetaClient();
-
-    assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
-    assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
-    assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
-    client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    // Await till enough time passes such that the first 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
-      Thread.sleep(2000);
-    }
-    Future<List<WriteStatus>> commit4 = service.submit(() -> writeBatch(new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
-        "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
-        HoodieJavaWriteClient::bulkInsert, false, 100, 100, 0, true));
-    Future<HoodieCleanMetadata> clean1 = service.submit(() -> new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)).clean());
-    commit4.get();
-    clean1.get();
-    client.close();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 2);
-    // Since we write rollbacks not clean, there should be no clean action on the timeline
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(CLEAN_ACTION)).countInstants() == 0);
-    assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    service.shutdown();
-  }
-
-  private Pair<StoragePath, List<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
-      throws Exception {
-    HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
-        .build()) : (getConfigBuilder().withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
-            .withOptimisticConsistencyGuardSleepTimeMs(1).build())
-        .build());
-    HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
-
-    client.startCommitWithTime(instantTime);
-    List<HoodieRecord> writeRecords = dataGen.generateInserts(instantTime, 200);
-    List<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-
-    // Create a dummy marker file to simulate the case that a marker file was created without data file.
-    // This should fail the commit
-    String partitionPath;
-    String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
-    FileSystem fs = (FileSystem) storage.getFileSystem();
-    if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
-      String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
-              markerFolderPath, storage, context, 1).values().stream()
-          .flatMap(Collection::stream).findFirst().get();
-      partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
-    } else {
-      partitionPath = Arrays
-          .stream(fs.globStatus(new Path(String.format("%s/*/*/*/*", markerFolderPath)),
-              path -> path.toString().contains(HoodieTableMetaClient.MARKER_EXTN)))
-          .limit(1).map(status -> status.getPath().getParent().toString()).collect(Collectors.toList()).get(0);
-    }
-
-    Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
-            cfg.getMarkersType(), getHoodieTable(metaClient, cfg), instantTime)
-        .create(partitionPath,
-            FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()),
-            IOType.MERGE);
-    if (!enableOptimisticConsistencyGuard) {
-      Exception e = assertThrows(HoodieCommitException.class, () -> {
-        client.commit(instantTime, result);
-      }, "Commit should fail due to consistency check");
-      assertTrue(e.getCause() instanceof HoodieIOException);
-    } else {
-      // with optimistic CG, commit should succeed
-      client.commit(instantTime, result);
-    }
-    return Pair.of(markerFilePath.get(), result);
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
-    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
-                                                      boolean populateMetaFields, Properties props) {
-    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
-    if (!populateMetaFields) {
-      builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.SIMPLE).build());
-    }
-    return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
-            .compactionSmallFileSize(smallFileSize)
-            // Set rollback to LAZY so no inflights are deleted
-            .insertSplitSize(insertSplitSize).build())
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
-            .build())
-        .withStorageConfig(HoodieStorageConfig.newBuilder()
-            .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
-            .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
-        .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
-        .withProps(props)
-        .build();
-  }
-
-  private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
-    Properties properties = new Properties();
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
-    if (!populateMetaFields) {
-      getPropertiesForKeyGen(populateMetaFields).entrySet().forEach(kv ->
-          properties.put(kv.getKey(), kv.getValue()));
-    }
-    return getConfigBuilder()
-        .withEmbeddedTimelineServerEnabled(false)
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(cleaningPolicy)
-            .withAutoClean(false).build())
-        .withTimelineLayoutVersion(1)
-        .withHeartbeatIntervalInMs(3 * 1000)
-        .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
-            .withRemoteServerPort(timelineServicePort).build())
-        .withAutoCommit(false)
-        .withLockConfig(HoodieLockConfig.newBuilder()
-            .withLockProvider(InProcessLockProvider.class)
-            .build())
-        .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
-        .withProperties(properties).build();
+    super.testParallelInsertAndCleanPreviousFailedCommits(true);
   }
 
   public static class WriteClientBrokenClustering<T extends HoodieRecordPayload> extends org.apache.hudi.client.HoodieJavaWriteClient<T> {
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index 520e5ba0a1f..7aac330a99d 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -17,6 +17,7 @@
 
 package org.apache.hudi.testutils;
 
+import org.apache.hudi.client.BaseHoodieWriteClient;
 import org.apache.hudi.client.HoodieJavaWriteClient;
 import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.client.common.HoodieJavaEngineContext;
@@ -50,7 +51,6 @@ import org.apache.hudi.common.table.view.SyncableFileSystemView;
 import org.apache.hudi.common.table.view.TableFileSystemView;
 import org.apache.hudi.common.testutils.HoodieTestTable;
 import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.util.FileFormatUtils;
 import org.apache.hudi.common.util.HoodieTimer;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
@@ -63,6 +63,7 @@ import org.apache.hudi.exception.HoodieMetadataException;
 import org.apache.hudi.index.HoodieIndex;
 import org.apache.hudi.index.JavaHoodieIndexFactory;
 import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
 import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
 import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
 import org.apache.hudi.metadata.HoodieTableMetadata;
@@ -106,6 +107,8 @@ import java.util.stream.Stream;
 import static org.apache.hudi.common.testutils.HoodieTestUtils.RAW_TRIPS_TEST_NAME;
 import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
 import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForKeys;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForRecords;
 import static org.apache.hudi.testutils.GenericRecordValidationTestUtils.readHFile;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -121,10 +124,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
 
   private static final Logger LOG = LoggerFactory.getLogger(HoodieJavaClientTestHarness.class);
 
-  protected StorageConfiguration<Configuration> storageConf;
   protected HoodieJavaEngineContext context;
   protected TestJavaTaskContextSupplier taskContextSupplier;
-  protected HoodieStorage storage;
   protected ExecutorService executorService;
   protected HoodieTableFileSystemView tableView;
   protected HoodieJavaWriteClient writeClient;
@@ -239,6 +240,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
     }
   }
 
+  @Override
   protected HoodieJavaWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
     if (null != writeClient) {
       writeClient.close();
@@ -264,6 +266,20 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
     return HoodieTableMetadata.create(engineContext, metaClient.getStorage(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
   }
 
+  @Override
+  protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
+    client.startCommitWithTime(commitTime);
+    List<WriteStatus> statusRDD = ((HoodieJavaWriteClient) client).upsert(inserts, commitTime);
+    if (autoCommitOff) {
+      client.commit(commitTime, statusRDD);
+    }
+    List<WriteStatus> statuses = statusRDD;
+    assertNoWriteErrors(statuses);
+    verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig(),
+        HoodieAvroKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
+    return statuses;
+  }
+
   /**
    * Validate the metadata tables contents to ensure it matches what is on the file system.
    */
@@ -739,6 +755,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
    * @param wrapped      Actual Records Generation function
    * @return Wrapped Function
    */
+  @Override
   public Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI,
                                                                               HoodieWriteConfig writeConfig,
                                                                               Function2<List<HoodieRecord>, String, Integer> wrapped) {
@@ -1027,12 +1044,4 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
     return builder;
   }
 
-  public static FileFormatUtils getFileUtilsInstance(HoodieTableMetaClient metaClient) {
-    return HoodieIOFactory.getIOFactory(metaClient.getStorage())
-        .getFileFormatUtils(metaClient.getTableConfig().getBaseFileFormat());
-  }
-
-  protected HoodieTableMetaClient createMetaClient() {
-    return HoodieTestUtils.createMetaClient(storageConf, basePath);
-  }
 }
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
index 9ed2dce3ce5..f5fa70c6668 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
@@ -265,7 +265,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase {
     // Initial inserts with TRIP_EXAMPLE_SCHEMA
     int numRecords = 10;
     insertFirstBatch(hoodieWriteConfig, client, "001", initCommitTime,
-                     numRecords, SparkRDDWriteClient::insert, false, true, numRecords);
+        numRecords, SparkRDDWriteClient::insert, false, true, numRecords);
     checkReadRecords("000", numRecords);
 
     // Updates with same schema is allowed
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 35ffd1035e1..b73ed53e62f 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -2285,7 +2285,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
     tableType = HoodieTableType.COPY_ON_WRITE;
     init(tableType);
     context = new HoodieSparkEngineContext(jsc);
-    HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+    HoodieWriteConfig config = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
     SparkRDDWriteClient client = getHoodieWriteClient(config);
 
     // Write 1 (Bulk insert)
@@ -2356,7 +2356,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
     tableType = HoodieTableType.COPY_ON_WRITE;
     init(tableType);
     context = new HoodieSparkEngineContext(jsc);
-    HoodieWriteConfig initialConfig = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+    HoodieWriteConfig initialConfig = getSmallInsertWriteConfigForMDT(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
     HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
         .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build()).build();
     SparkRDDWriteClient client = getHoodieWriteClient(config);
@@ -3278,23 +3278,6 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
     doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
   }
 
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
-    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER);
-    return builder.withCompactionConfig(
-            HoodieCompactionConfig.newBuilder()
-                .compactionSmallFileSize(smallFileSize)
-                // Set rollback to LAZY so no inflights are deleted
-                .insertSplitSize(insertSplitSize).build())
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build())
-        .withStorageConfig(
-            HoodieStorageConfig.newBuilder()
-                .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
-                .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
-        .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
-        .build();
-  }
-
   public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, HoodieIndex.IndexType indexType,
                                                     HoodieFailedWritesCleaningPolicy cleaningPolicy) {
     Properties properties = getDisabledRowWriterProperties();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 335a0f8f86a..834616ee804 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -18,7 +18,6 @@
 
 package org.apache.hudi.client.functional;
 
-import org.apache.hudi.avro.model.HoodieCleanMetadata;
 import org.apache.hudi.avro.model.HoodieClusteringPlan;
 import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
 import org.apache.hudi.client.BaseHoodieWriteClient;
@@ -29,104 +28,74 @@ import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.client.clustering.plan.strategy.SparkSingleFileSortPlanStrategy;
 import org.apache.hudi.client.clustering.run.strategy.SparkSingleFileSortExecutionStrategy;
 import org.apache.hudi.client.clustering.update.strategy.SparkRejectUpdateStrategy;
+import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
 import org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass;
 import org.apache.hudi.client.transaction.PreferWriterConflictResolutionStrategy;
 import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy;
-import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
 import org.apache.hudi.client.validator.SparkPreCommitValidator;
 import org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator;
 import org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator;
 import org.apache.hudi.common.config.HoodieMetadataConfig;
-import org.apache.hudi.common.config.HoodieStorageConfig;
-import org.apache.hudi.common.config.LockConfiguration;
-import org.apache.hudi.common.config.TypedProperties;
 import org.apache.hudi.common.data.HoodieData;
 import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.ConsistencyGuardConfig;
 import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.FileSlice;
-import org.apache.hudi.common.model.HoodieAvroRecord;
 import org.apache.hudi.common.model.HoodieBaseFile;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.model.HoodieFileGroupId;
 import org.apache.hudi.common.model.HoodieKey;
-import org.apache.hudi.common.model.HoodieOperation;
-import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
 import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordDelegate;
 import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
 import org.apache.hudi.common.model.HoodieWriteStat;
-import org.apache.hudi.common.model.IOType;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.marker.MarkerType;
 import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
 import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
 import org.apache.hudi.common.testutils.FileCreateUtils;
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestTable;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.testutils.RawTripTestPayload;
 import org.apache.hudi.common.util.ClusteringUtils;
-import org.apache.hudi.common.util.CollectionUtils;
 import org.apache.hudi.common.util.FileFormatUtils;
-import org.apache.hudi.common.util.MarkerUtils;
 import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieArchivalConfig;
 import org.apache.hudi.config.HoodieCleanConfig;
 import org.apache.hudi.config.HoodieClusteringConfig;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
 import org.apache.hudi.config.HoodieLockConfig;
 import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.data.HoodieJavaRDD;
 import org.apache.hudi.exception.HoodieClusteringException;
-import org.apache.hudi.exception.HoodieCommitException;
 import org.apache.hudi.exception.HoodieCorruptedDataException;
 import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.exception.HoodieInsertException;
 import org.apache.hudi.exception.HoodieUpsertException;
 import org.apache.hudi.exception.HoodieValidationException;
 import org.apache.hudi.exception.HoodieWriteConflictException;
 import org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner;
-import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.index.HoodieIndex.IndexType;
 import org.apache.hudi.io.HoodieMergeHandle;
 import org.apache.hudi.keygen.BaseKeyGenerator;
 import org.apache.hudi.keygen.KeyGenerator;
 import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
 import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
 import org.apache.hudi.table.BulkInsertPartitioner;
 import org.apache.hudi.table.HoodieSparkCopyOnWriteTable;
 import org.apache.hudi.table.HoodieSparkTable;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.HoodieWriteMetadata;
 import org.apache.hudi.table.action.cluster.ClusteringTestUtils;
-import org.apache.hudi.table.action.commit.HoodieWriteHelper;
-import org.apache.hudi.table.marker.WriteMarkersFactory;
 import org.apache.hudi.testutils.HoodieClientTestBase;
 import org.apache.hudi.testutils.HoodieClientTestUtils;
 import org.apache.hudi.testutils.HoodieSparkWriteableTestTable;
 
 import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.Path;
+import org.apache.spark.api.java.AbstractJavaRDDLike;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
-import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -138,7 +107,6 @@ import org.junit.jupiter.params.provider.ValueSource;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -146,27 +114,19 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY;
 import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGER;
-import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
 import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
 import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
 import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
-import static org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION;
 import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
 import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
 import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
 import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
-import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
 import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
 import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys;
 import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet;
@@ -174,26 +134,22 @@ import static org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENA
 import static org.apache.hudi.config.HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME;
 import static org.apache.hudi.config.HoodieClusteringConfig.UPDATES_STRATEGY;
 import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadata;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 @SuppressWarnings("unchecked")
 @Tag("functional")
 public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
-  private static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE";
   private static final Map<String, String> STRATEGY_PARAMS = new HashMap<String, String>() {
     {
       put("sortColumn", "record_key");
     }
   };
-
   private static Stream<Arguments> smallInsertHandlingParams() {
     return Arrays.stream(new Boolean[][] {{true}, {false}}).map(Arguments::of);
   }
@@ -202,14 +158,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     return Arrays.stream(new Boolean[][] {{true}, {false}}).map(Arguments::of);
   }
 
-  private static Stream<Arguments> rollbackFailedCommitsParams() {
-    return Stream.of(
-        Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, true),
-        Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, true),
-        Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, false)
-    );
-  }
-
   private static Stream<Arguments> rollbackAfterConsistencyCheckFailureParams() {
     return Stream.of(
         Arguments.of(true, true),
@@ -219,22 +167,102 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     );
   }
 
-  private HoodieTestTable testTable;
-
   private static final String COUNT_SQL_QUERY_FOR_VALIDATION = "select count(*) from <TABLE_NAME>";
 
+  private static HoodiePreCommitValidatorConfig createPreCommitValidatorConfig(int expectedNumberOfRows) {
+    return HoodiePreCommitValidatorConfig.newBuilder()
+            .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
+            .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + expectedNumberOfRows)
+            .build();
+  }
+
+  private static HoodieClusteringConfig.Builder createClusteringBuilder(boolean isInline, int inlineNumCommits) {
+    return HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0)
+            .withInlineClustering(isInline).withInlineClusteringNumCommits(inlineNumCommits)
+            .fromProperties(getDisabledRowWriterProperties());
+  }
+
+  private static HoodieLockConfig createLockConfig(ConflictResolutionStrategy conflictResolutionStrategy) {
+    return HoodieLockConfig.newBuilder().withLockProvider(FileSystemBasedLockProviderTestClass.class)
+            .withConflictResolutionStrategy(conflictResolutionStrategy).build();
+  }
+
+  private static HoodieCleanConfig createCleanConfig(HoodieFailedWritesCleaningPolicy policy, boolean autoClean) {
+    return HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(policy).withAutoClean(autoClean).build();
+  }
+
   @BeforeEach
   public void setUpTestTable() {
     testTable = HoodieSparkWriteableTestTable.of(metaClient);
   }
 
+  private final Function<List, JavaRDD> list2Rdd = recordList -> jsc.parallelize(recordList, 1);
+
+  private final Function<JavaRDD, List> rdd2List = AbstractJavaRDDLike::collect;
+
+  private final Function<HoodieWriteConfig, BaseHoodieWriteClient> createBrokenClusteringClient =
+      config -> new WriteClientBrokenClustering<>(context, config);
+
+  private final Function<HoodieWriteMetadata, HoodieWriteMetadata<List<WriteStatus>>> clusteringMetadataRdd2List =
+      metadata -> metadata.clone(((JavaRDD)(metadata.getWriteStatuses())).collect());
+
+  private final Function<HoodieWriteConfig, KeyGenerator> createKeyGenerator =
+      config -> HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps());
+
+  private final Function2<HoodieTable, HoodieTableMetaClient, HoodieWriteConfig> getHoodieTable =
+      (metaClient, config) -> getHoodieTable(metaClient, config);
+
+  @Override
+  protected Object castInsertFirstBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                        String initCommitTime, int numRecordsInThisCommit,
+                                        Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                        boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception {
+    return insertFirstBatch(writeConfig, (SparkRDDWriteClient) client, newCommitTime, initCommitTime, numRecordsInThisCommit,
+        (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+        isPreppedAPI, assertForCommit, expRecordsInThisCommit, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castWriteBatch(BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                  Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                  Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
+                                  Function3<Object, BaseHoodieWriteClient, Object, String> writeFn,
+                                  boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit,
+                                  boolean filterForCommitTimeWithAssert) throws Exception {
+    return writeBatch((SparkRDDWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+        (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime),
+        assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castUpdateBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime, String prevCommitTime,
+                                   Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
+                                   Function3<Object, BaseHoodieWriteClient, Object, String> writeFn, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+                                   boolean filterForCommitTimeWithAssert) throws Exception {
+    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
+        generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates);
+
+    return writeBatch((SparkRDDWriteClient) client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction,
+        (writeClient, records, commitTime) -> (JavaRDD<WriteStatus>) writeFn.apply(writeClient, records, commitTime), assertForCommit, expRecordsInThisCommit, expTotalRecords,
+        expTotalCommits, false, filterForCommitTimeWithAssert);
+  }
+
+  @Override
+  protected Object castDeleteBatch(HoodieWriteConfig writeConfig, BaseHoodieWriteClient client, String newCommitTime,
+                                   String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI,
+                                   boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception {
+    return deleteBatch(writeConfig, (SparkRDDWriteClient) client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit,
+        isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, filterForCommitTimeWithAssert);
+  }
+
   /**
    * Test Auto Commit behavior for HoodieWriteClient insert API.
    */
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testAutoCommitOnInsert(boolean populateMetaFields) throws Exception {
-    testAutoCommit(SparkRDDWriteClient::insert, false, populateMetaFields);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insert(recordRDD, instantTime), false, populateMetaFields);
   }
 
   /**
@@ -242,7 +270,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testAutoCommitOnInsertPrepped() throws Exception {
-    testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true, true);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.insertPreppedRecords(recordRDD, instantTime), true, true);
   }
 
   /**
@@ -251,7 +279,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testAutoCommitOnUpsert(boolean populateMetaFields) throws Exception {
-    testAutoCommit(SparkRDDWriteClient::upsert, false, populateMetaFields);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), false, populateMetaFields);
   }
 
   /**
@@ -260,7 +288,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testAutoCommitOnUpsertPrepped(boolean populateMetaFields) throws Exception {
-    testAutoCommit(SparkRDDWriteClient::upsertPreppedRecords, true, populateMetaFields);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, populateMetaFields);
   }
 
   /**
@@ -269,7 +297,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testAutoCommitOnBulkInsert(boolean populateMetaFields) throws Exception {
-    testAutoCommit(SparkRDDWriteClient::bulkInsert, false, populateMetaFields);
+    testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsert(recordRDD, instantTime), false, populateMetaFields);
   }
 
   /**
@@ -281,42 +309,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
         Option.empty()), true, true);
   }
 
-  /**
-   * Test auto-commit by applying the write function.
-   *
-   * @param writeFn One of HoodieWriteClient Write API
-   * @throws Exception in case of failure
-   */
-  private void testAutoCommit(Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn,
-                              boolean isPrepped, boolean populateMetaFields) throws Exception {
-    // Set autoCommit false
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
-    try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());) {
-
-      String prevCommitTime = "000";
-      String newCommitTime = "001";
-      int numRecords = 200;
-      JavaRDD<WriteStatus> result = insertFirstBatch(cfgBuilder.build(), client, newCommitTime, prevCommitTime, numRecords, writeFn,
-          isPrepped, false, numRecords);
-
-      assertFalse(testTable.commitExists(newCommitTime),
-          "If Autocommit is false, then commit should not be made automatically");
-      assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
-      assertTrue(testTable.commitExists(newCommitTime),
-          "After explicit commit, commit file should be created");
-    }
-  }
-
   @Test
   public void testPreCommitValidatorsOnInsert() throws Exception {
     int numRecords = 200;
-    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + numRecords)
-        .build();
     HoodieWriteConfig config = getConfigBuilder().withAutoCommit(true)
-        .withPreCommitValidatorConfig(validatorConfig).build();
+        .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200)).build();
     try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
       Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
           writeClient.bulkInsert(recordRDD, instantTime, Option.empty());
@@ -330,12 +327,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @Test
   public void testPreCommitValidationFailureOnInsert() throws Exception {
     int numRecords = 200;
-    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
-        //set wrong value for expected number of rows
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + 500)
-        .build();
-    HoodieWriteConfig config = getConfigBuilder().withPreCommitValidatorConfig(validatorConfig).build();
+    //set wrong value for expected number of rows
+    HoodieWriteConfig config = getConfigBuilder().withPreCommitValidatorConfig(createPreCommitValidatorConfig(500)).build();
     String newCommitTime = metaClient.createNewInstantTime();
     try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
       Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) ->
@@ -357,14 +350,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @Test
   public void testPreCommitValidationWithMultipleInflights() throws Exception {
     int numRecords = 200;
-    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
-        //set wrong value for expected number of rows
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + 500)
-        .build();
     HoodieWriteConfig config = getConfigBuilder()
-        .withCleanConfig(HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.NEVER).build())
-        .withPreCommitValidatorConfig(validatorConfig)
+        .withCleanConfig(createCleanConfig(HoodieFailedWritesCleaningPolicy.NEVER, true))
+        //set wrong value for expected number of rows
+        .withPreCommitValidatorConfig(createPreCommitValidatorConfig(500))
         .build();
 
     String instant1 = getHoodieWriteClient(config).createNewInstantTime();
@@ -383,14 +372,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertTrue(testTable.inflightCommitExists(instant1));
 
     numRecords = 300;
-    validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName())
-        //set wrong value for expected number of rows
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + numRecords)
-        .build();
     config = getConfigBuilder()
-        .withCleanConfig(HoodieCleanConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.NEVER).build())
-        .withPreCommitValidatorConfig(validatorConfig)
+        .withCleanConfig(createCleanConfig(HoodieFailedWritesCleaningPolicy.NEVER, true))
+        .withPreCommitValidatorConfig(createPreCommitValidatorConfig(numRecords))
         .build();
     String instant2 = getHoodieWriteClient(config).createNewInstantTime();
     // expect pre-commit validators to succeed. Note that validator is expected to exclude inflight instant1
@@ -414,7 +398,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testDeduplicationOnInsert(boolean populateMetaFields) throws Exception {
-    testDeduplication(SparkRDDWriteClient::insert, populateMetaFields);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.insert(list2Rdd.apply(records), commitTime)), populateMetaFields, false);
   }
 
   /**
@@ -423,7 +407,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testDeduplicationKeepOperationFieldOnInsert(boolean populateMetaFields) throws Exception {
-    testDeduplicationKeepOperation(SparkRDDWriteClient::insert, populateMetaFields);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.insert(list2Rdd.apply(records), commitTime)), populateMetaFields, true);
   }
 
   /**
@@ -432,7 +416,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testDeduplicationOnBulkInsert(boolean populateMetaFields) throws Exception {
-    testDeduplication(SparkRDDWriteClient::bulkInsert, populateMetaFields);
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.bulkInsert(list2Rdd.apply(records), commitTime)), populateMetaFields, false);
   }
 
   /**
@@ -440,145 +424,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testDeduplicationOnUpsert() throws Exception {
-    testDeduplication(SparkRDDWriteClient::upsert, true);
-  }
-
-  /**
-   * Test Deduplication Logic for write function.
-   *
-   * @param writeFn One of HoodieWriteClient non-prepped write APIs
-   * @throws Exception in case of failure
-   */
-  private void testDeduplication(
-      Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
-    String newCommitTime = "001";
-
-    String recordKey = UUID.randomUUID().toString();
-    HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
-    HoodieRecord<RawTripTestPayload> recordOne =
-        new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime));
-
-    HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
-    HoodieRecord recordTwo =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
-    // Same key and partition as keyTwo
-    HoodieRecord recordThree =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime));
-
-    HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieJavaRDD.of(
-        jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1));
-    HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
-        .combineInput(true, true);
-    addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
-    HoodieWriteConfig writeConfig = configBuilder.build();
-
-    // Global dedup should be done based on recordKey only
-    HoodieIndex index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(true);
-    int dedupParallelism = records.getNumPartitions() + 2;
-    HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(dedupParallelism, dedupedRecsRdd.getNumPartitions());
-    assertEquals(1, dedupedRecs.size());
-    assertEquals(dedupedRecs.get(0).getPartitionPath(), recordThree.getPartitionPath());
-    assertNodupesWithinPartition(dedupedRecs);
-
-    // non-Global dedup should be done based on both recordKey and partitionPath
-    index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(false);
-    dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(dedupParallelism, dedupedRecsRdd.getNumPartitions());
-    assertEquals(2, dedupedRecs.size());
-    assertNodupesWithinPartition(dedupedRecs);
-
-    // Perform write-action and check
-    JavaRDD<HoodieRecord> recordList = jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1);
-
-    try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
-      client.startCommitWithTime(newCommitTime);
-      List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime).collect();
-      assertNoWriteErrors(statuses);
-      assertEquals(2, statuses.size());
-      assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
-          .collect(Collectors.toList()));
-    }
-  }
-
-  /**
-   * Test Deduplication Logic for write function.
-   *
-   * @param writeFn One of HoodieWriteClient non-prepped write APIs
-   * @throws Exception in case of failure
-   */
-  private void testDeduplicationKeepOperation(
-      Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean populateMetaFields) throws Exception {
-    String newCommitTime = "001";
-
-    String recordKey = UUID.randomUUID().toString();
-    HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01");
-    HoodieRecord<RawTripTestPayload> recordOne =
-        new HoodieAvroRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime), HoodieOperation.INSERT);
-
-    HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01");
-    HoodieRecord recordTwo =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.INSERT);
-
-    // Same key and partition as keyTwo
-    HoodieRecord recordThree =
-        new HoodieAvroRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime), HoodieOperation.UPDATE_AFTER);
-
-    HoodieData<HoodieRecord<RawTripTestPayload>> records = HoodieJavaRDD.of(
-        jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1));
-    HoodieWriteConfig.Builder configBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAllowOperationMetadataField(true)
-        .combineInput(true, true);
-    addConfigsForPopulateMetaFields(configBuilder, populateMetaFields);
-    HoodieWriteConfig writeConfig = configBuilder.build();
-
-    // Global dedup should be done based on recordKey only
-    HoodieIndex index = mock(HoodieIndex.class);
-    when(index.isGlobal()).thenReturn(true);
-    int dedupParallelism = records.getNumPartitions() + 100;
-    HoodieData<HoodieRecord<RawTripTestPayload>> dedupedRecsRdd =
-        (HoodieData<HoodieRecord<RawTripTestPayload>>) HoodieWriteHelper.newInstance()
-            .deduplicateRecords(records, index, dedupParallelism, writeConfig.getSchema(), writeConfig.getProps(), HoodiePreCombineAvroRecordMerger.INSTANCE);
-    List<HoodieRecord<RawTripTestPayload>> dedupedRecs = dedupedRecsRdd.collectAsList();
-    assertEquals(dedupedRecs.get(0).getOperation(), recordThree.getOperation());
-
-    // Perform write-action and check
-    JavaRDD<HoodieRecord> recordList = jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1);
-
-    try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
-      client.startCommitWithTime(newCommitTime);
-      List<WriteStatus> statuses = writeFn.apply(client, recordList, newCommitTime).collect();
-      assertNoWriteErrors(statuses);
-      assertEquals(2, statuses.size());
-      assertNoDuplicatesInPartition(statuses.stream().map(WriteStatus::getWrittenRecordDelegates).flatMap(Collection::stream)
-          .collect(Collectors.toList()));
-    }
-  }
-
-  /**
-   * Assert that there is no duplicate key at the partition level.
-   *
-   * @param recordDelegates List of Hoodie record delegates
-   */
-  void assertNoDuplicatesInPartition(List<HoodieRecordDelegate> recordDelegates) {
-    Map<String, Set<String>> partitionToKeys = new HashMap<>();
-    for (HoodieRecordDelegate r : recordDelegates) {
-      String recordKey = r.getRecordKey();
-      String partitionPath = r.getPartitionPath();
-      if (!partitionToKeys.containsKey(partitionPath)) {
-        partitionToKeys.put(partitionPath, new HashSet<>());
-      }
-      assertFalse(partitionToKeys.get(partitionPath).contains(recordKey), "key " + recordKey + " is duplicate within partition " + partitionPath);
-      partitionToKeys.get(partitionPath).add(recordKey);
-    }
+    testDeduplication((client, records, commitTime) -> (List<WriteStatus>) rdd2List.apply((JavaRDD<WriteStatus>)client.upsert(list2Rdd.apply(records), commitTime)), true, false);
   }
 
   /**
@@ -587,9 +433,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testUpserts(boolean populateMetaFields) throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
-    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
-    testUpsertsInternal(cfgBuilder.build(), SparkRDDWriteClient::upsert, false);
+    testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsert(recordRDD, instantTime), populateMetaFields, false);
   }
 
   /**
@@ -597,132 +441,27 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testUpsertsPrepped() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testUpsertsInternal(cfgBuilder.build(), SparkRDDWriteClient::upsertPreppedRecords, true);
+    testUpsertsInternal((writeClient, recordRDD, instantTime) -> writeClient.upsertPreppedRecords(recordRDD, instantTime), true, true);
   }
 
-  /**
-   * Test one of HoodieWriteClient upsert(Prepped) APIs.
-   *
-   * @param config  Write Config
-   * @param writeFn One of Hoodie Write Function API
-   * @throws Exception in case of error
-   */
-  private void testUpsertsInternal(HoodieWriteConfig config,
-                                   Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn, boolean isPrepped)
-      throws Exception {
-    // Force using older timeline layout
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY)
-        .withRollbackUsingMarkers(true)
-        .withProps(config.getProps()).withTimelineLayoutVersion(
-            VERSION_0).build();
-
-    HoodieTableMetaClient.withPropertyBuilder()
-        .fromMetaClient(metaClient)
-        .setTimelineLayoutVersion(VERSION_0)
-        .setPopulateMetaFields(config.populateMetaFields())
-        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
-    SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String newCommitTime = "001";
-    String initCommitTime = "000";
-    int numRecords = 200;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert,
-        isPrepped, true, numRecords, config.populateMetaFields());
-
-    // Write 2 (updates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    numRecords = 100;
-    String commitTimeBetweenPrevAndNew = "002";
-    updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime,
-        Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, writeFn, isPrepped, true,
-        numRecords, 200, 2, config.populateMetaFields());
-
-    // Delete 1
-    prevCommitTime = newCommitTime;
-    newCommitTime = "005";
-    numRecords = 50;
-
-    deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecords, isPrepped, true,
-        0, 150, config.populateMetaFields());
-
-    // Now simulate an upgrade and perform a restore operation
-    HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion(
-        TimelineLayoutVersion.CURR_VERSION).build();
-    client = getHoodieWriteClient(newConfig);
-
-    client.savepoint("004", "user1", "comment1");
-
-    client.restoreToInstant("004", config.isMetadataTableEnabled());
-
-    assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
-
-    client.deleteSavepoint("004");
-    assertFalse(metaClient.reloadActiveTimeline().getSavePointTimeline().containsInstant("004"));
-
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(200,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + 200 + " records");
-
-    // Perform Delete again on upgraded dataset.
-    prevCommitTime = newCommitTime;
-    newCommitTime = "006";
-    numRecords = 50;
-
-    deleteBatch(newConfig, client, newCommitTime, prevCommitTime,
-        initCommitTime, numRecords, isPrepped, true,
-        0, 150);
-
-    HoodieActiveTimeline activeTimeline = new HoodieActiveTimeline(metaClient, false);
-    List<HoodieInstant> instants = activeTimeline.getCommitAndReplaceTimeline().getInstants();
-    assertEquals(5, instants.size());
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "001"),
-        instants.get(0));
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "004"),
-        instants.get(1));
-    // New Format should have all states of instants
-    assertEquals(new HoodieInstant(REQUESTED, COMMIT_ACTION, "006"),
-        instants.get(2));
-    assertEquals(new HoodieInstant(INFLIGHT, COMMIT_ACTION, "006"),
-        instants.get(3));
-    assertEquals(new HoodieInstant(COMPLETED, COMMIT_ACTION, "006"),
-        instants.get(4));
-
-    final HoodieWriteConfig cfg = hoodieWriteConfig;
+  @Override
+  protected void testMergeHandle(HoodieWriteConfig config) throws IOException {
     final String instantTime = "007";
     HoodieTableMetaClient metaClient = HoodieClientTestUtils.createMetaClient(jsc, basePath);
-    String basePathStr = basePath;
-    HoodieTable table = getHoodieTable(metaClient, cfg);
-    String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
+    HoodieTable table = getHoodieTable(metaClient, config);
+    Pair<String, String> partitionAndBaseFilePaths = getPartitionAndBaseFilePathsFromLatestCommitMetadata(metaClient);
+    String partitionPath = partitionAndBaseFilePaths.getLeft();
+    String baseFilePath = partitionAndBaseFilePaths.getRight();
     jsc.parallelize(Arrays.asList(1)).map(e -> {
-      HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
-          .fromBytes(metaClient.getActiveTimeline().getInstantDetails(
-                  metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get()).get(),
-              HoodieCommitMetadata.class);
-      String filePath = commitMetadata.getPartitionToWriteStats().values().stream()
-          .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
-          .map(ee -> ee.getPath()).orElse(null);
-      String partitionPath = commitMetadata.getPartitionToWriteStats().values().stream()
-          .flatMap(w -> w.stream()).filter(s -> s.getPath().endsWith(extension)).findAny()
-          .map(ee -> ee.getPartitionPath()).orElse(null);
-      Path baseFilePath = new Path(basePathStr, filePath);
-      HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath.toString());
+
+      HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath);
 
       HoodieMergeHandle handle = null;
       try {
-        handle = new HoodieMergeHandle(cfg, instantTime, table, new HashMap<>(),
-            partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new SparkTaskContextSupplier(),
+        handle = new HoodieMergeHandle(config, instantTime, table, new HashMap<>(),
+            partitionPath, FSUtils.getFileId(baseFile.getFileName()), baseFile, new SparkTaskContextSupplier(),
             config.populateMetaFields() ? Option.empty() :
-                Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+                Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps())));
         WriteStatus writeStatus = new WriteStatus(false, 0.0);
         writeStatus.setStat(new HoodieWriteStat());
         writeStatus.getStat().setNumWrites(0);
@@ -738,12 +477,12 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
       handle = null;
       try {
         final String newInstantTime = "006";
-        cfg.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
-        HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(cfg.getProps()).build();
+        config.getProps().setProperty("hoodie.merge.data.validation.enabled", "true");
+        HoodieWriteConfig cfg2 = HoodieWriteConfig.newBuilder().withProps(config.getProps()).build();
         handle = new HoodieMergeHandle(cfg2, newInstantTime, table, new HashMap<>(),
-            partitionPath, FSUtils.getFileId(baseFilePath.getName()), baseFile, new SparkTaskContextSupplier(),
+            partitionPath, FSUtils.getFileId(baseFile.getFileName()), baseFile, new SparkTaskContextSupplier(),
             config.populateMetaFields() ? Option.empty() :
-                Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()))));
+                Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps())));
         WriteStatus writeStatus = new WriteStatus(false, 0.0);
         writeStatus.setStat(new HoodieWriteStat());
         writeStatus.getStat().setNumWrites(0);
@@ -777,7 +516,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
         .fromMetaClient(metaClient)
         .setTimelineLayoutVersion(VERSION_0)
         .setPopulateMetaFields(config.populateMetaFields())
-        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
+        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePathV2().toString());
 
     SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
 
@@ -815,9 +554,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
     // verify that restore fails when "hoodie.archive.beyond.savepoint" is enabled.
     SparkRDDWriteClient finalClient = client;
-    assertThrows(IllegalArgumentException.class, () -> {
-      finalClient.restoreToSavepoint("004");
-    }, "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled");
+    assertThrows(IllegalArgumentException.class, () -> finalClient.restoreToSavepoint("004"),
+            "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled");
   }
 
   /**
@@ -826,9 +564,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testInsertsWithHoodieConcatHandle(boolean populateMetaFields) throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
-    testHoodieConcatHandle(cfgBuilder.build(), false);
+    testHoodieConcatHandle(populateMetaFields, false);
   }
 
   /**
@@ -836,49 +572,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    testHoodieConcatHandle(cfgBuilder.build(), true);
-  }
-
-  /**
-   * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API.
-   *
-   * @param config Write Config
-   * @throws Exception in case of error
-   */
-  private void testHoodieConcatHandle(HoodieWriteConfig config, boolean isPrepped)
-      throws Exception {
-    // Force using older timeline layout
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
-        .withProps(config.getProps()).withMergeAllowDuplicateOnInserts(true).withTimelineLayoutVersion(
-            VERSION_0).build();
-    HoodieTableMetaClient.withPropertyBuilder()
-        .fromMetaClient(metaClient)
-        .setTimelineLayoutVersion(VERSION_0)
-        .initTable(metaClient.getStorageConf().newInstance(), metaClient.getBasePath());
-
-    SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String newCommitTime = "001";
-    String initCommitTime = "000";
-    int numRecords = 200;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert,
-        isPrepped, true, numRecords, config.populateMetaFields());
-
-    // Write 2 (updates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    numRecords = 100;
-    String commitTimeBetweenPrevAndNew = "002";
-
-    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUniqueUpdates);
-
-    writeBatch(client, newCommitTime, prevCommitTime, Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime,
-        numRecords, recordGenFunction, SparkRDDWriteClient::insert, true, numRecords, 300,
-        2, false, config.populateMetaFields());
+    testHoodieConcatHandle(true, true);
   }
 
   /**
@@ -886,8 +580,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testInsertsWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), false);
+    testHoodieConcatHandleOnDupInserts(false);
   }
 
   /**
@@ -895,37 +588,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testInsertsPreppedWithHoodieConcatHandleOnDuplicateIncomingKeys() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
-    testHoodieConcatHandleOnDupInserts(cfgBuilder.build(), true);
-  }
-
-  private void testHoodieConcatHandleOnDupInserts(HoodieWriteConfig config, boolean isPrepped) throws Exception {
-    HoodieWriteConfig hoodieWriteConfig = getConfigBuilder()
-        .withProps(config.getProps())
-        .withMergeAllowDuplicateOnInserts(true)
-        .build();
-
-    SparkRDDWriteClient<RawTripTestPayload> client = getHoodieWriteClient(hoodieWriteConfig);
-
-    // Write 1 (only inserts)
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-    int firstInsertRecords = 50;
-    insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, firstInsertRecords, SparkRDDWriteClient::insert,
-        isPrepped, true, firstInsertRecords, config.populateMetaFields());
-
-    // Write 2 (updates with duplicates)
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    int secondInsertRecords = 100; // needs to be larger than firstInsertRecords to guarantee duplicate keys
-    List<String> commitTimesBetweenPrevAndNew = Arrays.asList("002", "003");
-
-    final Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        generateWrapRecordsFn(isPrepped, hoodieWriteConfig, dataGen::generateUpdates);
-
-    writeBatch(client, newCommitTime, prevCommitTime, Option.of(commitTimesBetweenPrevAndNew), initCommitTime,
-        secondInsertRecords, recordGenFunction, SparkRDDWriteClient::insert, true, secondInsertRecords,
-        firstInsertRecords + secondInsertRecords, 2, false, config.populateMetaFields());
+    testHoodieConcatHandleOnDupInserts(true);
   }
 
   @Test
@@ -949,12 +612,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     StoragePath completeRestoreFile = null;
     StoragePath backupCompletedRestoreFile = null;
     try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
-      final String commitTime1 = "001";
-      client.startCommitWithTime(commitTime1);
-      List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, 100);
-      JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
-      List<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1).collect();
-      assertNoWriteErrors(statuses);
+      insertBatchRecords(client, "001", 100, 3, 2, SparkRDDWriteClient::insert);
 
       // inject a pending restore
       client.savepoint("001", "user1", "comment1");
@@ -965,11 +623,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
           metaClient.reloadActiveTimeline().getRestoreTimeline().filterCompletedInstants()
               .getInstants().get(0);
       completeRestoreFile = new StoragePath(
-          config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
-              + "/" + restoreCompleted.getFileName());
+          config.getBasePath() + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME
+              + StoragePath.SEPARATOR + restoreCompleted.getFileName());
       backupCompletedRestoreFile = new StoragePath(
-          config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
-              + "/" + restoreCompleted.getFileName() + ".backup");
+          config.getBasePath() + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME
+              + StoragePath.SEPARATOR + restoreCompleted.getFileName() + ".backup");
       metaClient.getStorage().rename(completeRestoreFile, backupCompletedRestoreFile);
     }
 
@@ -982,14 +640,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     metaClient.getStorage().rename(backupCompletedRestoreFile, completeRestoreFile);
 
     // retrigger a new commit, should succeed.
-
     try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
-      final String commitTime3 = "003";
-      client.startCommitWithTime(commitTime3);
-      List<HoodieRecord> inserts3 = dataGen.generateInserts(commitTime3, 100);
-      JavaRDD<HoodieRecord> insertRecordsRDD3 = jsc.parallelize(inserts3, 2);
-      List<WriteStatus> statuses = client.insert(insertRecordsRDD3, commitTime3).collect();
-      assertNoWriteErrors(statuses);
+      insertBatchRecords(client, "003", 100, 3, 2, SparkRDDWriteClient::insert);
     }
   }
 
@@ -998,47 +650,18 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testDeletes() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
-    /**
-     * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records
-     */
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-
-    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
-    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        (String instantTime, Integer numRecordsInThisCommit) -> {
-          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
-          List<HoodieRecord> fewRecordsForDelete = dataGen.generateDeletes(instantTime, 100);
-
-          recordsInFirstBatch.addAll(fewRecordsForInsert);
-          recordsInFirstBatch.addAll(fewRecordsForDelete);
-          return recordsInFirstBatch;
+    Function3<Function2<List<HoodieRecord>, String, Integer>, String, Integer, List<HoodieRecord>> secondBatchGenFn =
+        (String instantTime, Integer numRecordsInThisCommit, List<HoodieRecord> recordsInFirstBatch) -> {
+          List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
+          return (time, numRecords) -> {
+            List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
+            List<HoodieRecord> fewRecordsForUpdate = recordsInFirstBatch.subList(50, 100);
+            recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
+            recordsInSecondBatch.addAll(fewRecordsForUpdate);
+            return recordsInSecondBatch;
+          };
         };
-    writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
-        // unused as genFn uses hard-coded number of inserts/updates/deletes
-        -1, recordGenFunction, SparkRDDWriteClient::upsert, true, 200, 200, 1, false,
-        true);
-
-    /**
-     * Write 2 (deletes+writes).
-     */
-    String prevCommitTime = newCommitTime;
-    newCommitTime = "004";
-    final List<HoodieRecord> recordsInSecondBatch = new ArrayList<>();
-
-    recordGenFunction = (String instantTime, Integer numRecordsInThisCommit) -> {
-      List<HoodieRecord> fewRecordsForDelete = recordsInFirstBatch.subList(0, 50);
-      List<HoodieRecord> fewRecordsForUpdate = recordsInFirstBatch.subList(50, 100);
-      recordsInSecondBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
-      recordsInSecondBatch.addAll(fewRecordsForUpdate);
-      return recordsInSecondBatch;
-    };
-    writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 100, recordGenFunction,
-        SparkRDDWriteClient::upsert, true, 50, 150, 2, false,
-        true);
+    super.testDeletes(secondBatchGenFn, 100, 50, 150);
   }
 
   /**
@@ -1049,85 +672,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testDeletesForInsertsInSameBatch() throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
-    /**
-     * Write 200 inserts and issue deletes to a subset(50) of inserts.
-     */
-    String initCommitTime = "000";
-    String newCommitTime = "001";
-
-    final List<HoodieRecord> recordsInFirstBatch = new ArrayList<>();
-    Function2<List<HoodieRecord>, String, Integer> recordGenFunction =
-        (String instantTime, Integer numRecordsInThisCommit) -> {
-          List<HoodieRecord> fewRecordsForInsert = dataGen.generateInserts(instantTime, 200);
-          List<HoodieRecord> fewRecordsForDelete = fewRecordsForInsert.subList(40, 90);
-
-          recordsInFirstBatch.addAll(fewRecordsForInsert);
-          recordsInFirstBatch.addAll(dataGen.generateDeletesFromExistingRecords(fewRecordsForDelete));
-          return recordsInFirstBatch;
-        };
-
-    writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime,
-        -1, recordGenFunction, SparkRDDWriteClient::upsert, true, 150, 150, 1, false,
-        true);
-  }
-
-  private void assertPartitionPathRecordKeys(List<Pair<String, String>> expectedPartitionPathRecKeyPairs, String[] fullPartitionPaths) {
-    Dataset<Row> rows = getAllRows(fullPartitionPaths);
-    List<Pair<String, String>> actualPartitionPathRecKeyPairs = getActualPartitionPathAndRecordKeys(rows);
-    // verify all partitionpath, record key matches
-    assertActualAndExpectedPartitionPathRecordKeyMatches(expectedPartitionPathRecKeyPairs, actualPartitionPathRecKeyPairs);
-  }
-
-  private List<Pair<String, String>> getActualPartitionPathAndRecordKeys(Dataset<org.apache.spark.sql.Row> rows) {
-    List<Pair<String, String>> actualPartitionPathRecKeyPairs = new ArrayList<>();
-    for (Row row : rows.collectAsList()) {
-      actualPartitionPathRecKeyPairs
-          .add(Pair.of(row.getAs("_hoodie_partition_path"), row.getAs("_row_key")));
-    }
-    return actualPartitionPathRecKeyPairs;
-  }
-
-  private Dataset<org.apache.spark.sql.Row> getAllRows(String[] fullPartitionPaths) {
-    return HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths);
-  }
-
-  private String getFullPartitionPath(String relativePartitionPath) {
-    return getFullPartitionPaths(relativePartitionPath)[0];
-  }
-
-  private String[] getFullPartitionPaths(String... relativePartitionPaths) {
-    String[] fullPartitionPaths = new String[relativePartitionPaths.length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, relativePartitionPaths[i]);
-    }
-    return fullPartitionPaths;
+    super.testDeletesForInsertsInSameBatch();
   }
 
-  private void assertActualAndExpectedPartitionPathRecordKeyMatches(List<Pair<String, String>> expectedPartitionPathRecKeyPairs,
-                                                                    List<Pair<String, String>> actualPartitionPathRecKeyPairs) {
-    // verify all partitionpath, record key matches
-    assertEquals(expectedPartitionPathRecKeyPairs.size(), actualPartitionPathRecKeyPairs.size());
-    for (Pair<String, String> entry : actualPartitionPathRecKeyPairs) {
-      assertTrue(expectedPartitionPathRecKeyPairs.contains(entry));
-    }
-
-    for (Pair<String, String> entry : expectedPartitionPathRecKeyPairs) {
-      assertTrue(actualPartitionPathRecKeyPairs.contains(entry));
-    }
-  }
-
-  private Pair<List<WriteStatus>, List<HoodieRecord>> insertBatchRecords(SparkRDDWriteClient client, String commitTime,
-                                                                         Integer recordNum, int expectStatueSize) {
+  private Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> insertBatchRecords(SparkRDDWriteClient client, String commitTime,
+                                                                         Integer recordNum, int expectStatusSize, int numSlices,
+                                                                         Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn) throws IOException {
     client.startCommitWithTime(commitTime);
-    List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime, recordNum);
-    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
-    List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime).collect();
-    assertNoWriteErrors(statuses);
-    assertEquals(expectStatueSize, statuses.size(), "check expect statue size.");
-    return Pair.of(statuses, inserts1);
+    List<HoodieRecord> inserts = dataGen.generateInserts(commitTime, recordNum);
+    JavaRDD<HoodieRecord> insertRecordsRDD = jsc.parallelize(inserts, numSlices);
+    JavaRDD<WriteStatus> statuses = writeFn.apply(client, insertRecordsRDD, commitTime);
+    assertNoWriteErrors(statuses.collect());
+    assertEquals(expectStatusSize, statuses.count(), "check expect status size.");
+    return Pair.of(statuses, inserts);
   }
 
   @Test
@@ -1143,23 +700,20 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     HoodieSparkCopyOnWriteTable table = (HoodieSparkCopyOnWriteTable) HoodieSparkTable.create(config, context, metaClient);
 
     //1. insert to generate 2 file group
-    String commitTime1 = "001";
-    Pair<List<WriteStatus>, List<HoodieRecord>> upsertResult = insertBatchRecords(client, commitTime1, 600, 2);
+    Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> upsertResult = insertBatchRecords(client, "001", 600, 2, 1, SparkRDDWriteClient::upsert);
     List<HoodieRecord> inserts1 = upsertResult.getValue();
     List<String> fileGroupIds1 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
         .map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
     assertEquals(2, fileGroupIds1.size());
 
     // 2. generate clustering plan for fileGroupIds1 file groups
-    String commitTime2 = "002";
     List<List<FileSlice>> firstInsertFileSlicesList = table.getFileSystemView().getAllFileGroups(testPartitionPath)
         .map(fileGroup -> fileGroup.getAllFileSlices().collect(Collectors.toList())).collect(Collectors.toList());
-    List<FileSlice>[] fileSlices = (List<FileSlice>[]) firstInsertFileSlicesList.toArray(new List[firstInsertFileSlicesList.size()]);
-    createRequestedReplaceInstant(this.metaClient, commitTime2, fileSlices);
+    List<FileSlice>[] fileSlices = (List<FileSlice>[]) firstInsertFileSlicesList.toArray(new List[0]);
+    createRequestedReplaceInstant(this.metaClient, "002", fileSlices);
 
     // 3. insert one record with no updating reject exception, and not merge the small file, just generate a new file group
-    String commitTime3 = "003";
-    insertBatchRecords(client, commitTime3, 1, 1);
+    insertBatchRecords(client, "003", 1, 1, 1, SparkRDDWriteClient::upsert);
     List<String> fileGroupIds2 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
         .map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
     assertEquals(3, fileGroupIds2.size());
@@ -1167,8 +721,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     // 4. update one record for the clustering two file groups, throw reject update exception
     String commitTime4 = "004";
     client.startCommitWithTime(commitTime4);
-    List<HoodieRecord> insertsAndUpdates3 = new ArrayList<>();
-    insertsAndUpdates3.addAll(dataGen.generateUpdates(commitTime4, inserts1));
+    List<HoodieRecord> insertsAndUpdates3 = new ArrayList<>(dataGen.generateUpdates(commitTime4, inserts1));
     String assertMsg = String.format("Not allowed to update the clustering files in partition: %s "
         + "For pending clustering operations, we are not going to support update for now.", testPartitionPath);
     assertThrows(HoodieUpsertException.class, () -> {
@@ -1176,10 +729,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     }, assertMsg);
 
     // 5. insert one record with no updating reject exception, will merge the small file
-    String commitTime5 = "005";
-    List<WriteStatus> statuses = insertBatchRecords(client, commitTime5, 1, 1).getKey();
+    JavaRDD<WriteStatus> statuses = insertBatchRecords(client, "005", 1, 1, 1, SparkRDDWriteClient::upsert).getKey();
     fileGroupIds2.removeAll(fileGroupIds1);
-    assertEquals(fileGroupIds2.get(0), statuses.get(0).getFileId());
+    assertEquals(fileGroupIds2.get(0), statuses.collect().get(0).getFileId());
     List<String> firstInsertFileGroupIds4 = table.getFileSystemView().getAllFileGroups(testPartitionPath)
         .map(fileGroup -> fileGroup.getFileGroupId().getFileId()).collect(Collectors.toList());
     assertEquals(3, firstInsertFileGroupIds4.size());
@@ -1258,7 +810,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertNoWriteErrors(statuses);
 
     assertEquals(2, statuses.size(), "2 files needs to be committed.");
-    HoodieTableMetaClient metadata = createMetaClient(basePath);
+    HoodieTableMetaClient metadata = createMetaClient();
 
     HoodieTable table = getHoodieTable(metadata, config);
     BaseFileOnlyView fileSystemView = table.getBaseFileOnlyView();
@@ -1308,7 +860,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     final String testPartitionPath = "2016/09/26";
     final int insertSplitLimit = 100;
     // setup the small file handling params
-
     HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit, false, mergeAllowDuplicateInserts); // hold upto 200 records max
     dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
     SparkRDDWriteClient client = getHoodieWriteClient(config);
@@ -1316,14 +867,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
     // Inserts => will write file1
     String commitTime1 = "001";
-    client.startCommitWithTime(commitTime1);
-    List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, insertSplitLimit); // this writes ~500kb
-    Set<String> keys1 = recordsToRecordKeySet(inserts1);
-    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
-    List<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1).collect();
-    assertNoWriteErrors(statuses);
+    Pair<JavaRDD<WriteStatus>, List<HoodieRecord>> insertResult = insertBatchRecords(client, commitTime1,
+            insertSplitLimit, 1, 1, SparkRDDWriteClient::insert); // this writes ~500kb
+    List<WriteStatus> statuses = insertResult.getLeft().collect();
+    Set<String> keys1 = recordsToRecordKeySet(insertResult.getRight());
     assertPartitionMetadata(basePath, new String[] {testPartitionPath}, storage);
-    assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
     String file1 = statuses.get(0).getFileId();
     assertEquals(100,
         fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
@@ -1331,13 +879,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
     // Second, set of Inserts should just expand file1
     String commitTime2 = "002";
-    client.startCommitWithTime(commitTime2);
-    List<HoodieRecord> inserts2 = dataGen.generateInserts(commitTime2, 40);
-    Set<String> keys2 = recordsToRecordKeySet(inserts2);
-    JavaRDD<HoodieRecord> insertRecordsRDD2 = jsc.parallelize(inserts2, 1);
-    statuses = client.insert(insertRecordsRDD2, commitTime2).collect();
-    assertNoWriteErrors(statuses);
-    assertEquals(1, statuses.size(), "Just 1 file needs to be updated.");
+    insertResult = insertBatchRecords(client, commitTime2, 40, 1, 1, SparkRDDWriteClient::insert);
+    Set<String> keys2 = recordsToRecordKeySet(insertResult.getRight());
+    statuses = insertResult.getLeft().collect();
     assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
     assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
 
@@ -1356,18 +900,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
     // Lots of inserts such that file1 is updated and expanded, a new file2 is created.
     String commitTime3 = "003";
-    client.startCommitWithTime(commitTime3);
-    List<HoodieRecord> inserts3 = dataGen.generateInserts(commitTime3, 200);
-    JavaRDD<HoodieRecord> insertRecordsRDD3 = jsc.parallelize(inserts3, 1);
-    statuses = client.insert(insertRecordsRDD3, commitTime3).collect();
-    assertNoWriteErrors(statuses);
-    assertEquals(2, statuses.size(), "2 files needs to be committed.");
+    insertResult = insertBatchRecords(client, commitTime3, 200, 2, 1, SparkRDDWriteClient::insert);
+    statuses = insertResult.getLeft().collect();
     assertEquals(340,
         fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
             + fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
         "file should contain 340 records");
 
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
+    HoodieTableMetaClient metaClient = createMetaClient();
     HoodieTable table = getHoodieTable(metaClient, config);
     List<HoodieBaseFile> files = table.getBaseFileOnlyView()
         .getLatestBaseFilesBeforeOrOn(testPartitionPath, commitTime3).collect(Collectors.toList());
@@ -1378,7 +918,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
       assertEquals(commitTime3, file.getCommitTime(), "All files must be at commit 3");
       totalInserts += fileUtils.readAvroRecords(storage, new StoragePath(file.getPath())).size();
     }
-    assertEquals(totalInserts, inserts1.size() + inserts2.size() + inserts3.size(), "Total number of records must add up");
+    assertEquals(340, totalInserts, "Total number of records must add up");
   }
 
   /**
@@ -1437,14 +977,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertNoWriteErrors(statuses);
     assertEquals(0, statuses.size(), "Just 0 write status for delete.");
 
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(150,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + 150 + " records");
+    assertTheEntireDatasetHasAllRecordsStill(150);
 
     // delete another batch. previous delete commit should have persisted the schema. If not,
     // this will throw exception
@@ -1454,87 +987,32 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testSimpleClustering(boolean populateMetaFields) throws Exception {
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties())
-        .build();
-    testInsertAndClustering(clusteringConfig, populateMetaFields, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
+    testInsertAndClustering(createClusteringBuilder(true, 1).build(), populateMetaFields, true,
+            false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
   }
 
   @Test
   public void testAndValidateClusteringOutputFiles() throws IOException {
-    String partitionPath = "2015/03/16";
-    testInsertTwoBatches(true, partitionPath);
-
-    // Trigger clustering
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withEmbeddedTimelineServerEnabled(false).withAutoCommit(false)
-        .withClusteringConfig(HoodieClusteringConfig.newBuilder().withInlineClustering(true).withInlineClusteringNumCommits(2)
-            .fromProperties(getDisabledRowWriterProperties()).build());
-    try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
-      int numRecords = 200;
-      String newCommitTime = client.createNewInstantTime();
-      List<HoodieRecord> records1 = dataGen.generateInserts(newCommitTime, numRecords);
-      client.startCommitWithTime(newCommitTime);
-      JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
-      JavaRDD<WriteStatus> statuses = client.insert(insertRecordsRDD1, newCommitTime);
-      client.commit(newCommitTime, statuses);
-      List<WriteStatus> statusList = statuses.collect();
-      assertNoWriteErrors(statusList);
-
-      metaClient = HoodieTableMetaClient.reload(metaClient);
-      HoodieInstant replaceCommitInstant =
-          metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
-      HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
-          .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
-              HoodieReplaceCommitMetadata.class);
-
-      List<String> filesFromReplaceCommit = new ArrayList<>();
-      replaceCommitMetadata.getPartitionToWriteStats()
-          .forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
-
-      // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
-      List<StoragePathInfo> pathInfoList =
-          storage.listDirectEntries(new StoragePath(basePath + "/" + partitionPath));
-      List<String> clusteredFiles = pathInfoList.stream()
-          .filter(entry ->
-              entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
-          .map(pathInfo -> partitionPath + "/" + pathInfo.getPath().getName())
-          .collect(Collectors.toList());
-      assertEquals(clusteredFiles, filesFromReplaceCommit);
-    }
+    testAndValidateClusteringOutputFiles(createBrokenClusteringClient, createClusteringBuilder(true, 2).build(), list2Rdd, rdd2List);
   }
 
   @Test
   public void testRollbackOfRegularCommitWithPendingReplaceCommitInTimeline() throws Exception {
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties())
-        .build();
     // trigger clustering, but do not complete
-    testInsertAndClustering(clusteringConfig, true, false, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
+    testInsertAndClustering(createClusteringBuilder(true, 1).build(), true, false,
+            false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
 
     // trigger another partial commit, followed by valid commit. rollback of partial commit should succeed.
     HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
     SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
     String commitTime1 = client.createNewInstantTime();
-    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
-    client.startCommitWithTime(commitTime1);
-    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
-    JavaRDD<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1);
-    List<WriteStatus> statusList = statuses.collect();
-    assertNoWriteErrors(statusList);
+    insertBatchRecords(client, commitTime1, 200, 1, 2, SparkRDDWriteClient::upsert).getLeft();
 
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
+    HoodieTableMetaClient metaClient = createMetaClient();
     assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterInflightsAndRequested().countInstants());
 
     // trigger another commit. this should rollback latest partial commit.
-    records1 = dataGen.generateInserts(commitTime1, 200);
-    client.startCommitWithTime(commitTime1);
-    insertRecordsRDD1 = jsc.parallelize(records1, 2);
-    statuses = client.upsert(insertRecordsRDD1, commitTime1);
-    statusList = statuses.collect();
-    assertNoWriteErrors(statusList);
+    JavaRDD<WriteStatus> statuses = insertBatchRecords(client, commitTime1, 200, 1, 2, SparkRDDWriteClient::upsert).getLeft();
     client.commit(commitTime1, statuses);
     metaClient.reloadActiveTimeline();
     // rollback should have succeeded. Essentially, the pending clustering should not hinder the rollback of regular commits.
@@ -1544,46 +1022,17 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testInlineScheduleClustering(boolean scheduleInlineClustering) throws IOException {
-    testInsertTwoBatches(true);
-
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withAsyncClusteringMaxCommits(1).withInlineClustering(false).withScheduleInlineClustering(scheduleInlineClustering)
-        .build();
-
-    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
-        .withClusteringConfig(clusteringConfig)
-        .withProps(getPropertiesForKeyGen()).build();
-    SparkRDDWriteClient client = getHoodieWriteClient(config);
-    dataGen = new HoodieTestDataGenerator(new String[] {"2015/03/16"});
-    String commitTime1 = client.createNewInstantTime();
-    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
-    client.startCommitWithTime(commitTime1);
-    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
-    JavaRDD<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1);
-    List<WriteStatus> statusList = statuses.collect();
-    assertNoWriteErrors(statusList);
-    client.commit(commitTime1, statuses);
-
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
-        ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
-    if (scheduleInlineClustering) {
-      assertEquals(1, pendingClusteringPlans.size());
-    } else {
-      assertEquals(0, pendingClusteringPlans.size());
-    }
+    HoodieClusteringConfig clusteringConfig = createClusteringBuilder(false, 1)
+            .withAsyncClusteringMaxCommits(1).withScheduleInlineClustering(scheduleInlineClustering).build();
+    testInlineScheduleClustering(createBrokenClusteringClient, clusteringConfig, list2Rdd, rdd2List);
   }
 
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testClusteringWithSortColumns(boolean populateMetaFields) throws Exception {
     // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringSortColumns(populateMetaFields ? "_hoodie_record_key" : "_row_key")
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties())
-        .build();
+    HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
+        .withClusteringSortColumns(populateMetaFields ? "_hoodie_record_key" : "_row_key").build();
     testInsertAndClustering(clusteringConfig, populateMetaFields, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
   }
 
@@ -1591,12 +1040,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @MethodSource("populateMetaFieldsParams")
   public void testClusteringWithSortOneFilePerGroup(boolean populateMetaFields) throws Exception {
     // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
+    HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
         .withClusteringSortColumns("begin_lat,begin_lon")
         .withClusteringPlanStrategyClass(SparkSingleFileSortPlanStrategy.class.getName())
         .withClusteringExecutionStrategyClass(SparkSingleFileSortExecutionStrategy.class.getName())
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
-        .fromProperties(getDisabledRowWriterProperties())
         .build();
     // note that assertSameFileIds is true for this test because of the plan and execution strategy
     testInsertAndClustering(clusteringConfig, populateMetaFields, true, true, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
@@ -1605,17 +1052,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @Test
   public void testPendingClusteringRollback() throws Exception {
     boolean populateMetaFields = true;
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties()).build();
-
     // start clustering, but don't commit
-    List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, populateMetaFields, false);
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
-        ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
-    assertEquals(1, pendingClusteringPlans.size());
+    List<HoodieRecord> allRecords = testInsertAndClustering(createClusteringBuilder(true, 1).build(), populateMetaFields, false);
+    HoodieTableMetaClient metaClient = createMetaClient();
+    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans = getAndAssertPendingClusteringPlans(true, metaClient);
     HoodieInstant pendingClusteringInstant = pendingClusteringPlans.get(0).getLeft();
 
     // complete another commit after pending clustering
@@ -1635,7 +1075,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
     // delete rollback.completed instant to mimic failed rollback of clustering. and then trigger rollback of clustering again. same rollback instant should be used.
     HoodieInstant rollbackInstant = metaClient.getActiveTimeline().getRollbackTimeline().lastInstant().get();
-    FileCreateUtils.deleteRollbackCommit(metaClient.getBasePath(), rollbackInstant.getTimestamp());
+    FileCreateUtils.deleteRollbackCommit(metaClient.getBasePathV2().toString(), rollbackInstant.getTimestamp());
     metaClient.reloadActiveTimeline();
 
     // create replace commit requested meta file so that rollback will not throw FileNotFoundException
@@ -1645,7 +1085,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder()
         .setClusteringPlan(clusteringPlan).setOperationType(WriteOperationType.CLUSTER.name()).build();
 
-    FileCreateUtils.createRequestedReplaceCommit(metaClient.getBasePath(), pendingClusteringInstant.getTimestamp(), Option.of(requestedReplaceMetadata));
+    FileCreateUtils.createRequestedReplaceCommit(metaClient.getBasePathV2().toString(), pendingClusteringInstant.getTimestamp(), Option.of(requestedReplaceMetadata));
 
     // trigger clustering again. no new rollback instants should be generated.
     try {
@@ -1665,19 +1105,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ValueSource(booleans = {true, false})
   public void testInflightClusteringRollbackWhenUpdatesAllowed(boolean rollbackPendingClustering) throws Exception {
     // setup clustering config with update strategy to allow updates during ingestion
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder()
-        .withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0)
+    HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
         .withClusteringUpdatesStrategy("org.apache.hudi.client.clustering.update.strategy.SparkAllowUpdateStrategy")
-        .withRollbackPendingClustering(rollbackPendingClustering)
-        .withInlineClustering(true).withInlineClusteringNumCommits(1)
-        .fromProperties(getDisabledRowWriterProperties()).build();
+        .withRollbackPendingClustering(rollbackPendingClustering).build();
 
     // start clustering, but don't commit keep it inflight
     List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, true, false);
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
-        ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
-    assertEquals(1, pendingClusteringPlans.size());
+    HoodieTableMetaClient metaClient = createMetaClient();
+    List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans = getAndAssertPendingClusteringPlans(true, metaClient);
     HoodieInstant pendingClusteringInstant = pendingClusteringPlans.get(0).getLeft();
     assertEquals(pendingClusteringInstant.getState(), INFLIGHT);
 
@@ -1685,6 +1120,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(EAGER);
     addConfigsForPopulateMetaFields(cfgBuilder, true);
     cfgBuilder.withClusteringConfig(clusteringConfig);
+    cfgBuilder.withProperties(getPropertiesForKeyGen());
     HoodieWriteConfig config = cfgBuilder.build();
     SparkRDDWriteClient client = getHoodieWriteClient(config);
     String commitTime = client.createNewInstantTime();
@@ -1699,11 +1135,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
   @Test
   public void testClusteringWithFailingValidator() throws Exception {
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringSortColumns("_hoodie_record_key").withInlineClustering(true)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
-        .fromProperties(getDisabledRowWriterProperties()).build();
+    HoodieClusteringConfig clusteringConfig = createClusteringBuilder(true, 1)
+        .withClusteringSortColumns("_hoodie_record_key").withInlineClustering(true).build();
     try {
       testInsertAndClustering(clusteringConfig, true, true, false, FailingPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
       fail("expected pre-commit clustering validation to fail");
@@ -1714,12 +1147,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
   @Test
   public void testClusteringInvalidConfigForSqlQueryValidator() throws Exception {
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties()).build();
     try {
-      testInsertAndClustering(clusteringConfig, false, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), "", "");
+      testInsertAndClustering(createClusteringBuilder(true, 1).build(), false, true,
+              false, SqlQueryEqualityPreCommitValidator.class.getName(), "", "");
       fail("expected pre-commit clustering validation to fail because sql query is not configured");
     } catch (HoodieValidationException e) {
       // expected
@@ -1728,24 +1158,16 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
 
   @Test
   public void testClusteringInvalidConfigForSqlQuerySingleResultValidator() throws Exception {
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties()).build();
-
-    testInsertAndClustering(clusteringConfig, false, true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
+    testInsertAndClustering(createClusteringBuilder(true, 1).build(), false, true,
+            false, SqlQuerySingleResultPreCommitValidator.class.getName(),
         "", COUNT_SQL_QUERY_FOR_VALIDATION + "#400");
   }
 
   @Test
   public void testClusteringInvalidConfigForSqlQuerySingleResultValidatorFailure() throws Exception {
-    // setup clustering config.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
-        .fromProperties(getDisabledRowWriterProperties()).build();
-
     try {
-      testInsertAndClustering(clusteringConfig, false, true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
+      testInsertAndClustering(createClusteringBuilder(true, 1).build(), false,
+              true, false, SqlQuerySingleResultPreCommitValidator.class.getName(),
           "", COUNT_SQL_QUERY_FOR_VALIDATION + "#802");
       fail("expected pre-commit clustering validation to fail because of count mismatch. expect 400 rows, not 802");
     } catch (HoodieValidationException e) {
@@ -1760,136 +1182,16 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   private List<HoodieRecord> testInsertAndClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields,
                                                      boolean completeClustering, boolean assertSameFileIds, String validatorClasses,
                                                      String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation) throws Exception {
-    Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields);
-    testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords);
+    Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords = testInsertTwoBatches(populateMetaFields, createBrokenClusteringClient);
+    testClustering(clusteringConfig, populateMetaFields, completeClustering, assertSameFileIds, validatorClasses, sqlQueryForEqualityValidation,
+            sqlQueryForSingleResultValidation, allRecords, clusteringMetadataRdd2List, createKeyGenerator);
     return allRecords.getLeft().getLeft();
   }
 
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testFailWritesOnInlineTableServiceExceptions(boolean shouldFail) throws IOException {
-    try {
-      Properties properties = new Properties();
-      properties.setProperty("hoodie.fail.writes.on.inline.table.service.exception", String.valueOf(shouldFail));
-      properties.setProperty("hoodie.auto.commit", "false");
-      properties.setProperty("hoodie.clustering.inline.max.commits", "1");
-      properties.setProperty("hoodie.clustering.inline", "true");
-      testInsertTwoBatches(true, "2015/03/16", properties, true);
-      assertFalse(shouldFail);
-    } catch (HoodieException e) {
-      assertEquals(CLUSTERING_FAILURE, e.getMessage());
-      assertTrue(shouldFail);
-    }
-  }
-
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields) throws IOException {
-    return testInsertTwoBatches(populateMetaFields, "2015/03/16");
-  }
-
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath) throws IOException {
-    return testInsertTwoBatches(populateMetaFields, partitionPath, new Properties(), false);
-  }
-
-  /**
-   * This method returns following three items:
-   * 1. List of all HoodieRecord written in the two batches of insert.
-   * 2. Commit instants of the two batches.
-   * 3. List of new file group ids that were written in the two batches.
-   */
-  private Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> testInsertTwoBatches(boolean populateMetaFields, String partitionPath, Properties props,
-                                                                                                    boolean failInlineClustering) throws IOException {
-    // create config to not update small files.
-    HoodieWriteConfig config = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false, populateMetaFields,
-        populateMetaFields ? props : getPropertiesForKeyGen());
-    SparkRDDWriteClient client;
-    if (failInlineClustering) {
-      if (null != writeClient) {
-        writeClient.close();
-        writeClient = null;
-      }
-      client = new WriteClientBrokenClustering(context, config);
-    } else {
-      client = getHoodieWriteClient(config);
-    }
-
-    dataGen = new HoodieTestDataGenerator(new String[] {partitionPath});
-    String commitTime1 = client.createNewInstantTime();
-    List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
-    List<WriteStatus> statuses1 = writeAndVerifyBatch(client, records1, commitTime1, populateMetaFields, failInlineClustering);
-    Set<HoodieFileGroupId> fileIds1 = getFileGroupIdsFromWriteStatus(statuses1);
-
-    String commitTime2 = client.createNewInstantTime();
-    List<HoodieRecord> records2 = dataGen.generateInserts(commitTime2, 200);
-    List<WriteStatus> statuses2 = writeAndVerifyBatch(client, records2, commitTime2, populateMetaFields, failInlineClustering);
-    client.close();
-    Set<HoodieFileGroupId> fileIds2 = getFileGroupIdsFromWriteStatus(statuses2);
-    Set<HoodieFileGroupId> fileIdsUnion = new HashSet<>(fileIds1);
-    fileIdsUnion.addAll(fileIds2);
-    //verify new files are created for 2nd write
-    Set<HoodieFileGroupId> fileIdIntersection = new HashSet<>(fileIds1);
-    fileIdIntersection.retainAll(fileIds2);
-    assertEquals(0, fileIdIntersection.size());
-    return Pair.of(Pair.of(Stream.concat(records1.stream(), records2.stream()).collect(Collectors.toList()), Arrays.asList(commitTime1, commitTime2)), fileIdsUnion);
-  }
-
-  private void testClustering(HoodieClusteringConfig clusteringConfig, boolean populateMetaFields, boolean completeClustering, boolean assertSameFileIds,
-                              String validatorClasses, String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
-                              Pair<Pair<List<HoodieRecord>, List<String>>, Set<HoodieFileGroupId>> allRecords) throws IOException {
-
-    HoodieWriteConfig config = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY).withAutoCommit(false)
-        .withClusteringConfig(clusteringConfig)
-        .withProps(getPropertiesForKeyGen()).build();
-    HoodieWriteMetadata<JavaRDD<WriteStatus>> clusterMetadata =
-        performClustering(clusteringConfig, populateMetaFields, completeClustering, validatorClasses, sqlQueryForEqualityValidation, sqlQueryForSingleResultValidation, allRecords.getLeft());
-    if (assertSameFileIds) {
-      Set<HoodieFileGroupId> replacedFileIds = clusterMetadata.getWriteStats().get().stream()
-          .map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
-      Set<HoodieFileGroupId> insertedFileIds = allRecords.getRight();
-      assertEquals(insertedFileIds, replacedFileIds);
-    }
-    if (completeClustering) {
-      String clusteringCommitTime = metaClient.reloadActiveTimeline().getCompletedReplaceTimeline()
-          .getReverseOrderedInstants().findFirst().get().getTimestamp();
-      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft().getLeft(), clusterMetadata.getWriteStatuses().collect(), config);
-    }
-  }
-
-  private HoodieWriteMetadata<JavaRDD<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig,
-                                                                      boolean populateMetaFields,
-                                                                      boolean completeClustering,
-                                                                      String validatorClasses,
-                                                                      String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation,
-                                                                      Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException {
-    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses))
-        .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation)
-        .withPrecommitValidatorSingleResultSqlQueries(sqlQueryForSingleResultValidation)
-        .build();
-
-    HoodieWriteConfig config = getConfigBuilder().withAutoCommit(false)
-        .withPreCommitValidatorConfig(validatorConfig)
-        .withProps(populateMetaFields ? new Properties() : getPropertiesForKeyGen())
-        .withClusteringConfig(clusteringConfig).build();
-
-    // create client with new config.
-    SparkRDDWriteClient client = getHoodieWriteClient(config);
-    String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
-    HoodieWriteMetadata<JavaRDD<WriteStatus>> clusterMetadata = client.cluster(clusteringCommitTime, completeClustering);
-    if (config.populateMetaFields()) {
-      verifyRecordsWrittenWithPreservedMetadata(new HashSet<>(allRecords.getRight()), allRecords.getLeft(), clusterMetadata.getWriteStatuses().collect());
-    } else {
-      verifyRecordsWritten(clusteringCommitTime, populateMetaFields, allRecords.getLeft(), clusterMetadata.getWriteStatuses().collect(), config);
-    }
-
-    Set<HoodieFileGroupId> replacedFileIds = new HashSet<>();
-    clusterMetadata.getPartitionToReplaceFileIds().entrySet().forEach(partitionFiles ->
-        partitionFiles.getValue().stream().forEach(file ->
-            replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file))));
-    return clusterMetadata;
-  }
-
-  private Set<HoodieFileGroupId> getFileGroupIdsFromWriteStatus(List<WriteStatus> statuses) {
-    return statuses.stream().map(s -> new HoodieFileGroupId(s.getPartitionPath(), s.getFileId())).collect(Collectors.toSet());
+    testFailWritesOnInlineTableServiceExceptions(shouldFail, createBrokenClusteringClient);
   }
 
   /**
@@ -1926,8 +1228,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   private void verifyInsertOverwritePartitionHandling(int batch1RecordsCount, int batch2RecordsCount, boolean populateMetaFields) throws Exception {
     final String testPartitionPath = "americas";
     HoodieWriteConfig config = getSmallInsertWriteConfig(2000,
-        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
-            ? new Properties() : getPropertiesForKeyGen());
+        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, getPropertiesForKeyGen(populateMetaFields));
     SparkRDDWriteClient client = getHoodieWriteClient(config);
     dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
 
@@ -1940,19 +1241,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     String commitTime2 = "002";
     client.startCommitWithTime(commitTime2, REPLACE_COMMIT_ACTION);
     List<HoodieRecord> inserts2 = dataGen.generateInserts(commitTime2, batch2RecordsCount);
-    List<HoodieRecord> insertsAndUpdates2 = new ArrayList<>();
-    insertsAndUpdates2.addAll(inserts2);
+    List<HoodieRecord> insertsAndUpdates2 = new ArrayList<>(inserts2);
     JavaRDD<HoodieRecord> insertAndUpdatesRDD2 = jsc.parallelize(insertsAndUpdates2, 2);
     HoodieWriteResult writeResult = client.insertOverwrite(insertAndUpdatesRDD2, commitTime2);
     statuses = writeResult.getWriteStatuses().collect();
     assertNoWriteErrors(statuses);
 
     assertEquals(batch1Buckets, new HashSet<>(writeResult.getPartitionToReplaceFileIds().get(testPartitionPath)));
-    verifyRecordsWritten(commitTime2, populateMetaFields, inserts2, statuses, config);
+    verifyRecordsWritten(commitTime2, populateMetaFields, inserts2, statuses, config,
+        HoodieSparkKeyGeneratorFactory.createKeyGenerator(config.getProps()));
   }
 
   private Set<String> getFileIdsFromWriteStatus(List<WriteStatus> statuses) {
-    return statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
+    return statuses.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
   }
 
   /**
@@ -1986,8 +1287,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
     List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1).collect();
     assertNoWriteErrors(statuses);
-    Set<String> batchBuckets = statuses.stream().map(s -> s.getFileId()).collect(Collectors.toSet());
-    verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig());
+    Set<String> batchBuckets = statuses.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
+    verifyRecordsWritten(commitTime1, true, inserts1, statuses, client.getConfig(),
+        HoodieSparkKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
     return batchBuckets;
   }
 
@@ -2010,8 +1312,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   private void verifyDeletePartitionsHandling(int batch1RecordsCount, int batch2RecordsCount, int batch3RecordsCount,
                                               boolean populateMetaFields) throws Exception {
     HoodieWriteConfig config = getSmallInsertWriteConfig(2000,
-        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
-            ? new Properties() : getPropertiesForKeyGen());
+        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, getPropertiesForKeyGen(populateMetaFields));
     SparkRDDWriteClient client = getHoodieWriteClient(config);
     dataGen = new HoodieTestDataGenerator();
 
@@ -2061,78 +1362,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertEquals(0, baseFiles.size());
   }
 
-  /**
-   * Verify data in base files matches expected records and commit time.
-   */
-  private void verifyRecordsWritten(String commitTime, boolean populateMetadataField,
-                                    List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, HoodieWriteConfig config) throws IOException {
-    List<GenericRecord> records = new ArrayList<>();
-    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
-    if (config.populateMetaFields()) {
-      for (GenericRecord record : records) {
-        String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
-        assertEquals(commitTime,
-            record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
-        assertTrue(expectedKeys.contains(recordKey));
-      }
-    } else {
-      KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps()));
-      for (GenericRecord record : records) {
-        String recordKey = keyGenerator.getKey(record).getRecordKey();
-        if (!populateMetadataField) {
-          assertNull(record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD));
-        }
-        assertTrue(expectedKeys.contains(recordKey));
-      }
-    }
-  }
-
-  @NotNull
-  private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
-    for (WriteStatus status : allStatus) {
-      StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
-      records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
-    }
-    Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
-    assertEquals(records.size(), expectedKeys.size());
-    return expectedKeys;
-  }
-
-  private void verifyRecordsWrittenWithPreservedMetadata(Set<String> commitTimes, List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus) {
-    List<GenericRecord> records = new ArrayList<>();
-    Set<String> expectedKeys = verifyRecordKeys(expectedRecords, allStatus, records);
-    Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
-        .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
-    assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
-    Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
-    for (GenericRecord record : records) {
-      String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
-      assertTrue(expectedKeys.contains(recordKey));
-      String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
-      assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
-    }
-  }
-
-  private List<WriteStatus> writeAndVerifyBatch(SparkRDDWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields) throws IOException {
-    return writeAndVerifyBatch(client, inserts, commitTime, populateMetaFields, false);
-  }
-
-  private List<WriteStatus> writeAndVerifyBatch(SparkRDDWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) throws IOException {
-    client.startCommitWithTime(commitTime);
-    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts, 2);
-    JavaRDD<WriteStatus> statusRDD = client.upsert(insertRecordsRDD1, commitTime);
-    if (autoCommitOff) {
-      client.commit(commitTime, statusRDD);
-    }
-    List<WriteStatus> statuses = statusRDD.collect();
-    assertNoWriteErrors(statuses);
-    verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig());
-
-    return statuses;
-  }
-
   private Pair<Set<String>, List<HoodieRecord>> testUpdates(String instantTime, SparkRDDWriteClient client,
-                                                            int sizeToInsertAndUpdate, int expectedTotalRecords)
+                                                            int sizeToInsertAndUpdate, int expectedRecords)
       throws IOException {
     client.startCommitWithTime(instantTime);
     List<HoodieRecord> inserts = dataGen.generateInserts(instantTime, sizeToInsertAndUpdate);
@@ -2145,14 +1376,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     List<WriteStatus> statuses = client.upsert(insertAndUpdatesRDD, instantTime).collect();
     assertNoWriteErrors(statuses);
 
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(expectedTotalRecords,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + expectedTotalRecords + " records");
+    assertTheEntireDatasetHasAllRecordsStill(expectedRecords);
     return Pair.of(keys, inserts);
   }
 
@@ -2169,14 +1393,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
     assertEquals(existingFile, statuses.get(0).getFileId(), "Existing file should be expanded");
 
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(expectedRecords,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + expectedRecords + " records");
+    assertTheEntireDatasetHasAllRecordsStill(expectedRecords);
 
     StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
     assertEquals(expectedRecords,
@@ -2191,29 +1408,24 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     }
   }
 
+  @Override
+  protected String[] assertTheEntireDatasetHasAllRecordsStill(int expectedRecords) {
+    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
+    for (int i = 0; i < fullPartitionPaths.length; i++) {
+      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
+    }
+    assertEquals(expectedRecords,
+            HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(), "Must contain " + expectedRecords + " records");
+    return fullPartitionPaths;
+  }
+
   /**
    * Test delete with delete api.
    */
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testDeletesWithoutInserts(boolean populateMetaFields) {
-    final String testPartitionPath = "2016/09/26";
-    final int insertSplitLimit = 100;
-    // setup the small file handling params
-    HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit,
-        TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields
-            ? new Properties() : getPropertiesForKeyGen());
-    dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
-    SparkRDDWriteClient client = getHoodieWriteClient(config);
-
-    // delete non existent keys
-    String commitTime1 = "001";
-    client.startCommitWithTime(commitTime1);
-
-    List<HoodieRecord> dummyInserts = dataGen.generateInserts(commitTime1, 20);
-    List<HoodieKey> hoodieKeysToDelete = randomSelectAsHoodieKeys(dummyInserts, 20);
-    JavaRDD<HoodieKey> deleteKeys = jsc.parallelize(hoodieKeysToDelete, 1);
-    client.delete(deleteKeys, commitTime1).collect();
+    testDeletesWithoutInserts(populateMetaFields, list2Rdd, rdd2List);
   }
 
   /**
@@ -2221,47 +1433,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
    */
   @Test
   public void testCommitWritesRelativePaths() throws Exception {
-
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, true);
-    try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
-      HoodieTableMetaClient metaClient = createMetaClient(basePath);
-      HoodieSparkTable table = HoodieSparkTable.create(cfgBuilder.build(), context, metaClient);
-
-      String instantTime = "000";
-      client.startCommitWithTime(instantTime);
-
-      List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
-      JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
-
-      JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-
-      assertTrue(client.commit(instantTime, result), "Commit should succeed");
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-
-      // Get base file paths from commit metadata
-      String actionType = metaClient.getCommitActionType();
-      HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
-      HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
-      HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
-          .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(),
-              HoodieCommitMetadata.class);
-      String basePath = table.getMetaClient().getBasePath();
-      Collection<String> commitPathNames =
-          commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
-
-      // Read from commit file
-      HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
-          metaClient.reloadActiveTimeline()
-              .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
-          HoodieCommitMetadata.class);
-      HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
-      // Compare values in both to make sure they are equal.
-      for (String pathName : paths.values()) {
-        assertTrue(commitPathNames.contains(pathName));
-      }
-    }
+    testCommitWritesRelativePaths(list2Rdd);
   }
 
   /**
@@ -2270,60 +1442,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @MethodSource("populateMetaFieldsParams")
   public void testMetadataStatsOnCommit(boolean populateMetaFields) throws Exception {
-    HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
-    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
-    HoodieWriteConfig cfg = cfgBuilder.build();
-    SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-
-    String instantTime0 = "000";
-    client.startCommitWithTime(instantTime0);
-
-    List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
-    JavaRDD<HoodieRecord> writeRecords0 = jsc.parallelize(records0, 1);
-    JavaRDD<WriteStatus> result0 = client.bulkInsert(writeRecords0, instantTime0);
-
-    assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
-    assertTrue(testTable.commitExists(instantTime0),
-        "After explicit commit, commit file should be created");
-
-    // Read from commit file
-    HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
-        metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime0)).get(),
-        HoodieCommitMetadata.class);
-    int inserts = 0;
-    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
-      for (HoodieWriteStat stat : pstat.getValue()) {
-        inserts += stat.getNumInserts();
-      }
-    }
-    assertEquals(200, inserts);
-
-    // Update + Inserts such that they just expand file1
-    String instantTime1 = "001";
-    client.startCommitWithTime(instantTime1);
-
-    List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
-    JavaRDD<HoodieRecord> writeRecords1 = jsc.parallelize(records1, 1);
-    JavaRDD<WriteStatus> result1 = client.upsert(writeRecords1, instantTime1);
-
-    assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
-    assertTrue(testTable.commitExists(instantTime1),
-        "After explicit commit, commit file should be created");
-
-    // Read from commit file
-    metadata = HoodieCommitMetadata.fromBytes(
-        metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime1)).get(),
-        HoodieCommitMetadata.class);
-    inserts = 0;
-    int upserts = 0;
-    for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
-      for (HoodieWriteStat stat : pstat.getValue()) {
-        inserts += stat.getNumInserts();
-        upserts += stat.getNumUpdateWrites();
-      }
-    }
-    assertEquals(0, inserts);
-    assertEquals(200, upserts);
+    testMetadataStatsOnCommit(populateMetaFields, list2Rdd);
   }
 
   /**
@@ -2332,74 +1451,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-    String instantTime = "000";
-    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
-        .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
-    SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-    Pair<StoragePath, JavaRDD<WriteStatus>> result = testConsistencyCheck(
-        metaClient, instantTime, enableOptimisticConsistencyGuard);
-
-    // Delete orphan marker and commit should succeed
-    metaClient.getStorage().deleteFile(result.getKey());
-    if (!enableOptimisticConsistencyGuard) {
-      assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-      // Marker directory must be removed
-      assertFalse(metaClient.getStorage()
-          .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    } else {
-      // with optimistic, first client.commit should have succeeded.
-      assertTrue(testTable.commitExists(instantTime),
-          "After explicit commit, commit file should be created");
-      // Marker directory must be removed
-      assertFalse(metaClient.getStorage()
-          .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    }
+    testConsistencyCheckDuringFinalize(context, enableOptimisticConsistencyGuard, getHoodieTable, list2Rdd, rdd2List);
   }
 
   private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
                                                                      boolean populateMetaFields) throws Exception {
-    String instantTime = "00000000000010";
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
-    Properties properties = new Properties();
-    if (!populateMetaFields) {
-      properties = getPropertiesForKeyGen();
-    }
-
-    HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build() :
-        getConfigBuilder().withRollbackUsingMarkers(rollbackUsingMarkers).withAutoCommit(false)
-            .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
-                .withConsistencyCheckEnabled(true)
-                .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
-                .withOptimisticConsistencyGuardSleepTimeMs(1).build())
-            .withProperties(properties).build();
-    SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-    testConsistencyCheck(metaClient, instantTime, enableOptimisticConsistencyGuard);
-
-    if (!enableOptimisticConsistencyGuard) {
-      // Rollback of this commit should succeed with FailSafeCG
-      client.rollback(instantTime);
-      assertFalse(testTable.commitExists(instantTime),
-          "After explicit rollback, commit file should not be present");
-      // Marker directory must be removed after rollback
-      assertFalse(metaClient.getStorage().exists(
-          new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-    } else {
-      // if optimistic CG is enabled, commit should have succeeded.
-      assertTrue(testTable.commitExists(instantTime),
-          "With optimistic CG, first commit should succeed. commit file should be present");
-      // Marker directory must be removed after rollback
-      assertFalse(metaClient.getStorage().exists(
-          new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
-      client.rollback(instantTime);
-      assertFalse(testTable.commitExists(instantTime),
-          "After explicit rollback, commit file should not be present");
-    }
+    testRollbackAfterConsistencyCheckFailureUsingFileList(context, rollbackUsingMarkers, enableOptimisticConsistencyGuard,
+            populateMetaFields, getHoodieTable, list2Rdd, rdd2List);
   }
 
   @ParameterizedTest
@@ -2414,257 +1472,19 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     testRollbackAfterConsistencyCheckFailureUsingFileList(true, enableOptimisticConsistencyGuard, populateMetCols);
   }
 
-  //@ParameterizedTest
-  //@MethodSource("rollbackFailedCommitsParams")
   @Test
   public void testRollbackFailedCommits() throws Exception {
-    // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
-    boolean populateMetaFields = true;
-    HoodieTestUtils.init(storageConf, basePath);
-    SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-
-    // perform 1 successful commit
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-
-    // Perform 2 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "100",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // refresh data generator to delete records generated from failed commits
-    dataGen = new HoodieTestDataGenerator();
-    // Perform 1 successful write
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
-    assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
-    assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
-    assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
-    // Await till enough time passes such that the first 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
-      Thread.sleep(2000);
-    }
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    // Perform 1 successful write
-    writeBatch(client, "500", "400", Option.of(Arrays.asList("500")), "500",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-    client.clean();
-    client.close();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    if (cleaningPolicy.isLazy()) {
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
-              .countInstants()
-              == 2);
-      // Since we write rollbacks not clean, there should be no clean action on the timeline
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
-              .countInstants()
-              == 0);
-      assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    } else if (cleaningPolicy.isNever()) {
-      // never will get translated to Lazy if OCC is enabled.
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
-              .countInstants()
-              == 2);
-      // There should be no clean or rollback action on the timeline
-      assertTrue(
-          timeline
-              .getTimelineOfActions(CollectionUtils.createSet(CLEAN_ACTION))
-              .countInstants()
-              == 0);
-      assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    }
+    super.testRollbackFailedCommits(true);
   }
 
   @Test
   public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception {
-    boolean populateMetaFields = true;
-    HoodieTestUtils.init(storageConf, basePath);
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
-    SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    // Perform 1 successful writes to table
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, true);
-
-    // Perform 1 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Toggle cleaning policy to LAZY
-    cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
-    // Perform 2 failed writes to table
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Await till enough time passes such that the 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("400");
-      Thread.sleep(2000);
-    }
-    client.clean();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
-    // Perform 2 failed commits
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    writeBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
-        0, false);
-    client.close();
-    // Toggle cleaning policy to EAGER
-    cleaningPolicy = EAGER;
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
-    client.startCommit();
-    client.close();
-    timeline = metaClient.getActiveTimeline().reload();
-    // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
-    assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1);
+    super.testRollbackFailedCommitsToggleCleaningPolicy(true);
   }
 
   @Test
   public void testParallelInsertAndCleanPreviousFailedCommits() throws Exception {
-    HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY;
-    ExecutorService service = Executors.newFixedThreadPool(2);
-    HoodieTestUtils.init(storageConf, basePath);
-    SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    // perform 1 successful write
-    writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
-        0, true);
-
-    // Perform 2 failed writes to table
-    writeBatch(client, "200", "100", Option.of(Arrays.asList("200")), "200",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
-        0, false);
-    client.close();
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300",
-        100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
-        0, false);
-    client.close();
-    // refresh data generator to delete records generated from failed commits
-    dataGen = new HoodieTestDataGenerator();
-    // Create a successful commit
-    Future<JavaRDD<WriteStatus>> commit3 = service.submit(() -> writeBatch(new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
-        "400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
-        SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
-    commit3.get();
-    HoodieTableMetaClient metaClient = createMetaClient(basePath);
-
-    assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
-    assertTrue(metaClient.getActiveTimeline().filterInflights().countInstants() == 2);
-    assertTrue(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 2);
-    client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
-    // Await till enough time passes such that the first 2 failed commits heartbeats are expired
-    boolean conditionMet = false;
-    while (!conditionMet) {
-      conditionMet = client.getHeartbeatClient().isHeartbeatExpired("300");
-      Thread.sleep(2000);
-    }
-    Future<JavaRDD<WriteStatus>> commit4 = service.submit(() -> writeBatch(new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
-        "500", "400", Option.of(Arrays.asList("500")), "500", 100, dataGen::generateInserts,
-        SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
-    Future<HoodieCleanMetadata> clean1 = service.submit(() -> new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)).clean());
-    commit4.get();
-    clean1.get();
-    HoodieActiveTimeline timeline = metaClient.getActiveTimeline().reload();
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 2);
-    // Since we write rollbacks not clean, there should be no clean action on the timeline
-    assertTrue(timeline.getTimelineOfActions(
-        CollectionUtils.createSet(CLEAN_ACTION)).countInstants() == 0);
-    assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
-    client.close();
-  }
-
-  private Pair<StoragePath, JavaRDD<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
-      throws Exception {
-    HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withMaxConsistencyCheckIntervalMs(1).withInitialConsistencyCheckIntervalMs(1).withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build())
-        .build()) : (getConfigBuilder().withAutoCommit(false)
-        .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
-            .withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard)
-            .withOptimisticConsistencyGuardSleepTimeMs(1).build())
-        .build());
-    SparkRDDWriteClient client = getHoodieWriteClient(cfg);
-
-    client.startCommitWithTime(instantTime);
-    JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(instantTime, 200), 1);
-    JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-    result.collect();
-
-    // Create a dummy marker file to simulate the case that a marker file was created without data file.
-    // This should fail the commit
-    String partitionPath;
-    String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
-    if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
-      String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
-              markerFolderPath, storage, context, 1).values().stream()
-          .flatMap(Collection::stream).findFirst().get();
-      partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
-    } else {
-      partitionPath = storage.globEntries(
-              new StoragePath(String.format("%s/*/*/*/*", markerFolderPath)), path ->
-                  path.toString().contains(HoodieTableMetaClient.MARKER_EXTN))
-          .stream()
-          .limit(1).map(status -> status.getPath().getParent().toString())
-          .collect(Collectors.toList()).get(0);
-    }
-
-    Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
-            cfg.getMarkersType(), getHoodieTable(metaClient, cfg), instantTime)
-        .create(partitionPath,
-            FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(), BASE_FILE_EXTENSION),
-            IOType.MERGE);
-    if (!enableOptimisticConsistencyGuard) {
-      Exception e = assertThrows(HoodieCommitException.class, () -> {
-        client.commit(instantTime, result);
-      }, "Commit should fail due to consistency check");
-      assertTrue(e.getCause() instanceof HoodieIOException);
-    } else {
-      // with optimistic CG, commit should succeed
-      client.commit(instantTime, result);
-    }
-    return Pair.of(markerFilePath.get(), result);
+    super.testParallelInsertAndCleanPreviousFailedCommits(true);
   }
 
   @Test
@@ -2680,17 +1500,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1);
     JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, firstInstantTime);
     assertTrue(client.commit(firstInstantTime, result), "Commit should succeed");
-    assertTrue(testTable.commitExists(firstInstantTime),
-        "After explicit commit, commit file should be created");
+    assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created");
 
-    // Check the entire dataset has all records still
-    String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
-    for (int i = 0; i < fullPartitionPaths.length; i++) {
-      fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
-    }
-    assertEquals(numRecords,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + numRecords + " records");
+    String[] fullPartitionPaths = assertTheEntireDatasetHasAllRecordsStill(numRecords);
 
     String nextInstantTime = "0001";
     client.startCommitWithTime(nextInstantTime);
@@ -2699,24 +1511,18 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     JavaRDD<WriteStatus> inserts = client.bulkInsert(insertRecords, nextInstantTime);
     JavaRDD<WriteStatus> upserts = client.upsert(updateRecords, nextInstantTime);
     assertTrue(client.commit(nextInstantTime, inserts.union(upserts)), "Commit should succeed");
-    assertTrue(testTable.commitExists(firstInstantTime),
-        "After explicit commit, commit file should be created");
+    assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created");
     int totalRecords = 2 * numRecords;
     assertEquals(totalRecords,
-        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
-        "Must contain " + totalRecords + " records");
+        HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(), "Must contain " + totalRecords + " records");
   }
 
   @Test
   public void testClusteringCommitInPresenceOfInflightCommit() throws Exception {
     Properties properties = getDisabledRowWriterProperties();
     properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + "/.hoodie/.locks");
-    HoodieLockConfig lockConfig = HoodieLockConfig.newBuilder()
-        .withLockProvider(FileSystemBasedLockProviderTestClass.class)
-        .withConflictResolutionStrategy(new PreferWriterConflictResolutionStrategy())
-        .build();
-    HoodieCleanConfig cleanConfig = HoodieCleanConfig.newBuilder()
-        .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).withAutoClean(false).build();
+    HoodieLockConfig lockConfig = createLockConfig(new PreferWriterConflictResolutionStrategy());
+    HoodieCleanConfig cleanConfig = createCleanConfig(HoodieFailedWritesCleaningPolicy.LAZY, false);
     HoodieWriteConfig insertWriteConfig = getConfigBuilder().withAutoCommit(false)
         .withCleanConfig(cleanConfig)
         .withLockConfig(lockConfig)
@@ -2741,17 +1547,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
         2, false);
 
     // Schedule and execute a clustering plan on the same partition. During conflict resolution the commit should fail.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1)
-        .build();
-    HoodiePreCommitValidatorConfig preCommitValidatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(StringUtils.nullToEmpty(SqlQuerySingleResultPreCommitValidator.class.getName()))
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#200")
-        .build();
     HoodieWriteConfig clusteringWriteConfig = getConfigBuilder().withAutoCommit(false)
         .withCleanConfig(cleanConfig)
-        .withClusteringConfig(clusteringConfig)
-        .withPreCommitValidatorConfig(preCommitValidatorConfig)
+        .withClusteringConfig(createClusteringBuilder(true, 1).build())
+        .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200))
         .withLockConfig(lockConfig)
         .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
         .withProperties(properties)
@@ -2779,15 +1578,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
   public void testIngestionCommitInPresenceOfCompletedClusteringCommit() throws Exception {
     Properties properties = getDisabledRowWriterProperties();
     properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + "/.hoodie/.locks");
-    HoodieLockConfig lockConfig = HoodieLockConfig.newBuilder()
-        .withLockProvider(FileSystemBasedLockProviderTestClass.class)
-        .withConflictResolutionStrategy(new PreferWriterConflictResolutionStrategy())
-        .build();
-    HoodieCleanConfig cleanConfig = HoodieCleanConfig.newBuilder()
-        .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).withAutoClean(false).build();
+    HoodieCleanConfig cleanConfig = createCleanConfig(HoodieFailedWritesCleaningPolicy.LAZY, false);
     HoodieWriteConfig insertWriteConfig = getConfigBuilder().withAutoCommit(false)
         .withCleanConfig(cleanConfig)
-        .withLockConfig(lockConfig)
+        .withLockConfig(createLockConfig(new PreferWriterConflictResolutionStrategy()))
         .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
         .withProperties(properties)
         .build();
@@ -2809,27 +1603,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
         2, false);
 
     // Schedule and execute a clustering plan on the same partition. During conflict resolution the commit should fail.
-    HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
-        .withClusteringTargetPartitions(0)
-        .withInlineClusteringNumCommits(1)
-        .build();
-
     // Since it is harder to test corner cases where the ingestion writer is at dedupe step right before the inflight file creation
     // and clustering commit is just about to complete, using the default conflict resolutions strategy only for clustering job
     // To create a successful commit.
-    HoodieLockConfig clusteringLockConfig = HoodieLockConfig.newBuilder()
-        .withLockProvider(FileSystemBasedLockProviderTestClass.class)
-        .withConflictResolutionStrategy(new SimpleConcurrentFileWritesConflictResolutionStrategy())
-        .build();
-    HoodiePreCommitValidatorConfig preCommitValidatorConfig = HoodiePreCommitValidatorConfig.newBuilder()
-        .withPreCommitValidator(StringUtils.nullToEmpty(SqlQuerySingleResultPreCommitValidator.class.getName()))
-        .withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#200")
-        .build();
     HoodieWriteConfig clusteringWriteConfig = getConfigBuilder().withAutoCommit(false)
         .withCleanConfig(cleanConfig)
-        .withClusteringConfig(clusteringConfig)
-        .withPreCommitValidatorConfig(preCommitValidatorConfig)
-        .withLockConfig(clusteringLockConfig)
+        .withClusteringConfig(createClusteringBuilder(true, 1).build())
+        .withPreCommitValidatorConfig(createPreCommitValidatorConfig(200))
+        .withLockConfig(createLockConfig(new SimpleConcurrentFileWritesConflictResolutionStrategy()))
         .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
         .withProperties(properties)
         .build();
@@ -2847,75 +1628,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     assertThrows(HoodieWriteConflictException.class, () -> client.commit(inflightCommit, ingestionResult));
   }
 
-  /**
-   * Build Hoodie Write Config for small data file sizes.
-   */
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize) {
-    return getSmallInsertWriteConfig(insertSplitSize, false);
-  }
-
-  /**
-   * Build Hoodie Write Config for small data file sizes.
-   */
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema) {
-    return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, false);
-  }
-
-  /**
-   * Build Hoodie Write Config for small data file sizes.
-   */
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, boolean mergeAllowDuplicateInserts) {
-    return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, dataGen.getEstimatedFileSizeInBytes(150), mergeAllowDuplicateInserts);
-  }
-
-  /**
-   * Build Hoodie Write Config for specified small file sizes.
-   */
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize) {
-    return getSmallInsertWriteConfig(insertSplitSize, useNullSchema, smallFileSize, false);
-  }
-
-  /**
-   * Build Hoodie Write Config for specified small file sizes.
-   */
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema, long smallFileSize, boolean mergeAllowDuplicateInserts) {
-    String schemaStr = useNullSchema ? NULL_SCHEMA : TRIP_EXAMPLE_SCHEMA;
-    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts);
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize) {
-    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false);
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts) {
-    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, mergeAllowDuplicateInserts, true, new Properties());
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean populateMetaFields, Properties props) {
-    return getSmallInsertWriteConfig(insertSplitSize, schemaStr, smallFileSize, false, populateMetaFields, props);
-  }
-
-  private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, String schemaStr, long smallFileSize, boolean mergeAllowDuplicateInserts,
-                                                      boolean populateMetaFields, Properties props) {
-    HoodieWriteConfig.Builder builder = getConfigBuilder(schemaStr);
-    if (!populateMetaFields) {
-      builder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.SIMPLE).build());
-    }
-    return builder.withCompactionConfig(HoodieCompactionConfig.newBuilder()
-            .compactionSmallFileSize(smallFileSize)
-            // Set rollback to LAZY so no inflights are deleted
-            .insertSplitSize(insertSplitSize).build())
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
-            .build())
-        .withStorageConfig(HoodieStorageConfig.newBuilder()
-            .hfileMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200))
-            .parquetMaxFileSize(dataGen.getEstimatedFileSizeInBytes(200)).build())
-        .withMergeAllowDuplicateOnInserts(mergeAllowDuplicateInserts)
-        .withProps(props)
-        .build();
-  }
-
   protected HoodieInstant createRequestedReplaceInstant(HoodieTableMetaClient metaClient, String clusterTime, List<FileSlice>[] fileSlices) throws IOException {
     HoodieClusteringPlan clusteringPlan =
         ClusteringUtils.createClusteringPlan(EXECUTION_STRATEGY_CLASS_NAME.defaultValue(), STRATEGY_PARAMS, fileSlices, Collections.emptyMap());
@@ -2927,32 +1639,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
     return clusteringInstant;
   }
 
-  private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
-    Properties properties = new Properties();
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
-    properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
-    if (!populateMetaFields) {
-      getPropertiesForKeyGen(populateMetaFields).entrySet().forEach(kv ->
-          properties.put(kv.getKey(), kv.getValue()));
-    }
-    return getConfigBuilder()
-        .withEmbeddedTimelineServerEnabled(false)
-        .withCleanConfig(HoodieCleanConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(cleaningPolicy)
-            .withAutoClean(false).build())
-        .withTimelineLayoutVersion(1)
-        .withHeartbeatIntervalInMs(3 * 1000)
-        .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
-            .withRemoteServerPort(timelineServicePort).build())
-        .withAutoCommit(false)
-        .withLockConfig(HoodieLockConfig.newBuilder()
-            .withLockProvider(InProcessLockProvider.class)
-            .build())
-        .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
-        .withProperties(properties).build();
-  }
-
   public static class FailingPreCommitValidator<T extends HoodieRecordPayload, I, K, O extends HoodieData<WriteStatus>> extends SparkPreCommitValidator<T, I, K, O> {
 
     public FailingPreCommitValidator(HoodieSparkTable table, HoodieEngineContext context, HoodieWriteConfig config) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
index 2fceb1057da..107ce6a84de 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
@@ -52,6 +52,8 @@ import java.util.List;
 import java.util.function.Function;
 
 import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForKeys;
+import static org.apache.hudi.testutils.Assertions.assertPartitionMetadataForRecords;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -177,6 +179,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
    * @param wrapped Actual Records Generation function
    * @return Wrapped Function
    */
+  @Override
   public Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI,
       HoodieWriteConfig writeConfig,
       Function2<List<HoodieRecord>, String, Integer> wrapped) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index 5b1f514982b..eefa825bc5c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -21,9 +21,11 @@ import org.apache.hudi.HoodieConversionUtils;
 import org.apache.hudi.avro.model.HoodieActionInstant;
 import org.apache.hudi.avro.model.HoodieCleanMetadata;
 import org.apache.hudi.avro.model.HoodieCleanerPlan;
+import org.apache.hudi.client.BaseHoodieWriteClient;
 import org.apache.hudi.client.SparkRDDReadClient;
 import org.apache.hudi.client.SparkRDDWriteClient;
 import org.apache.hudi.client.SparkTaskContextSupplier;
+import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.client.common.HoodieSparkEngineContext;
 import org.apache.hudi.common.HoodieCleanStat;
 import org.apache.hudi.common.config.HoodieMetadataConfig;
@@ -52,13 +54,13 @@ import org.apache.hudi.data.HoodieJavaRDD;
 import org.apache.hudi.exception.HoodieMetadataException;
 import org.apache.hudi.hadoop.fs.HadoopFSUtils;
 import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
 import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
 import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
 import org.apache.hudi.metadata.HoodieTableMetadata;
 import org.apache.hudi.metadata.HoodieTableMetadataWriter;
 import org.apache.hudi.metadata.MetadataPartitionType;
 import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
-import org.apache.hudi.storage.HoodieStorage;
 import org.apache.hudi.storage.HoodieStorageUtils;
 import org.apache.hudi.storage.StorageConfiguration;
 import org.apache.hudi.storage.StoragePath;
@@ -70,7 +72,6 @@ import org.apache.hudi.timeline.service.TimelineService;
 import org.apache.hudi.util.JFunction;
 import org.apache.hudi.utils.HoodieWriterClientTestHarness;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
@@ -107,6 +108,7 @@ import scala.Tuple2;
 
 import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
 import static org.apache.hudi.common.util.CleanerUtils.convertCleanMetadata;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertLinesMatch;
@@ -129,9 +131,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
   protected JavaSparkContext jsc;
   protected HoodieSparkEngineContext context;
   protected SparkSession sparkSession;
-  protected StorageConfiguration<Configuration> storageConf;
   protected SQLContext sqlContext;
-  protected HoodieStorage storage;
   protected ExecutorService executorService;
   protected HoodieTableMetaClient metaClient;
   protected SparkRDDWriteClient writeClient;
@@ -398,6 +398,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
     return readClient;
   }
 
+  @Override
   public SparkRDDWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
     if (null != writeClient) {
       writeClient.close();
@@ -472,6 +473,22 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
     return Pair.of(partitionPathStatMap, globalStat);
   }
 
+  @Override
+  protected List<WriteStatus> writeAndVerifyBatch(BaseHoodieWriteClient client, List<HoodieRecord> inserts, String commitTime, boolean populateMetaFields, boolean autoCommitOff) {
+    client.startCommitWithTime(commitTime);
+    JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts, 2);
+    JavaRDD<WriteStatus> statusRDD = ((SparkRDDWriteClient) client).upsert(insertRecordsRDD1, commitTime);
+    if (autoCommitOff) {
+      client.commit(commitTime, statusRDD);
+    }
+    List<WriteStatus> statuses = statusRDD.collect();
+    assertNoWriteErrors(statuses);
+    verifyRecordsWritten(commitTime, populateMetaFields, inserts, statuses, client.getConfig(),
+        HoodieSparkKeyGeneratorFactory.createKeyGenerator(client.getConfig().getProps()));
+
+    return statuses;
+  }
+
   /**
    * Validate the metadata tables contents to ensure it matches what is on the file system.
    */
@@ -683,10 +700,6 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
     return new HoodieInstant(inflightOnly, "clean", instantTime);
   }
 
-  protected HoodieTableMetaClient createMetaClient(String basePath) {
-    return HoodieTestUtils.createMetaClient(storageConf, basePath);
-  }
-
   protected HoodieTableMetaClient createMetaClient(SparkSession spark, String basePath) {
     return HoodieClientTestUtils.createMetaClient(spark, basePath);
   }
diff --git a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
index a2786e0a179..4a2768119bf 100644
--- a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
+++ b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java
@@ -22,7 +22,6 @@ import org.apache.flink.test.util.AbstractTestBase;
 import org.apache.flink.types.Row;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.EnumSource;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
index e536e0c0853..87ce14f5c84 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
@@ -18,6 +18,7 @@
 
 package org.apache.hudi.common.testutils;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -26,6 +27,8 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.common.table.view.SyncableFileSystemView;
 import org.apache.hudi.exception.HoodieIOException;
 
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
 import org.junit.jupiter.api.io.TempDir;
 
 import java.io.IOException;
@@ -46,6 +49,9 @@ public class HoodieCommonTestHarness {
   @TempDir
   public java.nio.file.Path tempDir;
 
+  protected StorageConfiguration<Configuration> storageConf;
+  protected HoodieStorage storage;
+
   protected void setTableName(String tableName) {
     this.tableName = tableName;
   }
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
index 4192a47d51d..1c747962cf6 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceReadWithDeletes.java
@@ -54,6 +54,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.apache.hudi.common.table.HoodieTableConfig.TYPE;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -156,7 +157,7 @@ public class TestDataSourceReadWithDeletes extends SparkClientFunctionalTestHarn
     metaClient = HoodieTableMetaClient.reload(metaClient);
     client.startCommitWithTime(instant);
     List<WriteStatus> writeStatuses = client.upsert(writeRecords, instant).collect();
-    org.apache.hudi.testutils.Assertions.assertNoWriteErrors(writeStatuses);
+    assertNoWriteErrors(writeStatuses);
     metaClient = HoodieTableMetaClient.reload(metaClient);
     return writeStatuses;
   }