You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by zh...@apache.org on 2025/01/23 01:45:00 UTC

(hudi) branch incremental-TableService-CI3 updated: HUDI-8900

This is an automated email from the ASF dual-hosted git repository.

zhangyue19921010 pushed a commit to branch incremental-TableService-CI3
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/incremental-TableService-CI3 by this push:
     new 8eff45898bd HUDI-8900
8eff45898bd is described below

commit 8eff45898bdaf0dd748657ad6d7699d3da69c998
Author: YueZhang <zh...@163.com>
AuthorDate: Thu Jan 23 09:44:34 2025 +0800

    HUDI-8900
---
 .../org/apache/hudi/config/HoodieWriteConfig.java  |  16 ++
 .../action/BaseTableServicePlanActionExecutor.java | 200 +++++++++++++
 .../action/IncrementalPartitionAwareStrategy.java  |  48 ++++
 .../cluster/ClusteringPlanActionExecutor.java      |  11 +-
 .../cluster/ClusteringPlanPartitionFilter.java     |  26 +-
 ...sistentHashingBucketClusteringPlanStrategy.java |   5 +-
 .../cluster/strategy/ClusteringPlanStrategy.java   |   4 +-
 .../PartitionAwareClusteringPlanStrategy.java      |  96 +++++--
 .../compact/ScheduleCompactionActionExecutor.java  |   8 +-
 .../BaseHoodieCompactionPlanGenerator.java         |  24 +-
 .../generators/HoodieCompactionPlanGenerator.java  |  18 +-
 .../HoodieLogCompactionPlanGenerator.java          |  25 +-
 .../strategy/BoundedIOCompactionStrategy.java      |  21 +-
 .../BoundedPartitionAwareCompactionStrategy.java   |  17 +-
 .../compact/strategy/CompactionStrategy.java       |  29 +-
 .../strategy/CompositeCompactionStrategy.java      |  26 +-
 .../strategy/DayBasedCompactionStrategy.java       |  10 +-
 .../LogFileNumBasedCompactionStrategy.java         |  26 +-
 .../LogFileSizeBasedCompactionStrategy.java        |  29 +-
 .../PartitionRegexBasedCompactionStrategy.java     |   9 +-
 .../strategy/UnBoundedCompactionStrategy.java      |   9 +-
 .../UnBoundedPartitionAwareCompactionStrategy.java |  14 +-
 .../TestBaseTableServicePlanActionExecutor.java    | 308 +++++++++++++++++++++
 .../TestPartitionAwareClusteringPlanStrategy.java  |   7 -
 .../FlinkSizeBasedClusteringPlanStrategy.java      |   3 +-
 ...inkSizeBasedClusteringPlanStrategyRecently.java |   8 +-
 .../TestFlinkSizeBasedClusteringPlanStrategy.java  |   4 +-
 .../JavaSizeBasedClusteringPlanStrategy.java       |   3 +-
 .../strategy/SparkSingleFileSortPlanStrategy.java  |   6 +-
 .../SparkSizeBasedClusteringPlanStrategy.java      |   3 +-
 .../TestSparkSizeBasedClusteringPlanStrategy.java  |   2 +-
 .../table/action/cluster/ClusteringTestUtils.java  |  13 +
 .../action/cluster/TestIncrementalClustering.java  | 218 +++++++++++++++
 ...TestSparkBuildClusteringGroupsForPartition.java |   6 +-
 .../TestSparkClusteringPlanPartitionFilter.java    |  10 +-
 .../strategy/TestHoodieCompactionStrategy.java     | 180 ++++++++----
 .../src/main/avro/HoodieClusteringPlan.avsc        |  10 +-
 .../src/main/avro/HoodieCompactionOperation.avsc   |   8 +
 .../hudi/common/testutils/CompactionTestUtils.java |   2 +-
 .../hudi/common/testutils/FileCreateUtils.java     |   5 +
 .../org/apache/hudi/utils/TestClusteringUtil.java  |   2 +-
 .../org/apache/hudi/utils/TestCompactionUtil.java  |   2 +-
 .../hudi/common/testutils/HoodieTestTable.java     |  21 ++
 .../table/action/compact/TestHoodieCompactor.java  |  96 ++++++-
 .../strategy/SmallBoundedIOCompactionStrategy.java |  59 ++++
 .../hudi/procedure/TestClusteringProcedure.scala   |  28 +-
 46 files changed, 1466 insertions(+), 209 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index c7e14b6b4e1..0204f57c71e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -808,6 +808,13 @@ public class HoodieWriteConfig extends HoodieConfig {
       .withDocumentation("Avro schema of the partial updates. This is automatically set by the "
           + "Hudi write client and user is not expected to manually change the value.");
 
+  public static final ConfigProperty<Boolean> INCREMENTAL_TABLE_SERVICE_ENABLED = ConfigProperty
+      .key("hoodie.table.services.incremental.enabled")
+      .defaultValue(true)
+      .markAdvanced()
+      .sinceVersion("1.0.0")
+      .withDocumentation("Whether to enable incremental table service. So far Clustering and Compaction support incremental processing.");
+
   /**
    * Config key with boolean value that indicates whether record being written during MERGE INTO Spark SQL
    * operation are already prepped.
@@ -2179,6 +2186,10 @@ public class HoodieWriteConfig extends HoodieConfig {
     return getString(WRITE_PARTIAL_UPDATE_SCHEMA);
   }
 
+  public boolean isIncrementalTableServiceEnabled() {
+    return getBoolean(INCREMENTAL_TABLE_SERVICE_ENABLED);
+  }
+
   public double getParquetCompressionRatio() {
     return getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION);
   }
@@ -3341,6 +3352,11 @@ public class HoodieWriteConfig extends HoodieConfig {
       return this;
     }
 
+    public Builder withIncrementalTableServiceEnabled(boolean incrementalTableServiceEnabled) {
+      writeConfig.setValue(INCREMENTAL_TABLE_SERVICE_ENABLED, String.valueOf(incrementalTableServiceEnabled));
+      return this;
+    }
+
     protected void setDefaults() {
       writeConfig.setDefaultValue(MARKERS_TYPE, getDefaultMarkersType(engineType));
       // Check for mandatory properties
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java
new file mode 100644
index 00000000000..06779a3d5e8
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
+import org.apache.hudi.common.util.ClusteringUtils;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.table.HoodieTable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
+
+public abstract class BaseTableServicePlanActionExecutor<T, I, K, O, R> extends BaseActionExecutor<T, I, K, O, R> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BaseTableServicePlanActionExecutor.class);
+  private static final Set<String> MOR_COMMITS = CollectionUtils.createSet(DELTA_COMMIT_ACTION, REPLACE_COMMIT_ACTION);
+  private static final Set<String> COW_COMMITS = CollectionUtils.createSet(COMMIT_ACTION, REPLACE_COMMIT_ACTION);
+
+  public BaseTableServicePlanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config,
+                                            HoodieTable<T, I, K, O> table, String instantTime) {
+    super(context, config, table, instantTime);
+  }
+
+  /**
+   * Get partitions, if strategy implement `IncrementalPartitionAwareStrategy` then return incremental partitions,
+   * otherwise return all partitions of the table
+   * @param strategy
+   * @return
+   */
+  public List<String> getPartitions(Object strategy, TableServiceType type) {
+    if (config.isIncrementalTableServiceEnabled() && strategy instanceof IncrementalPartitionAwareStrategy) {
+      try {
+        // get incremental partitions.
+        LOG.info("Start to fetch incremental partitions for " + type);
+        Pair<Option<HoodieInstant>, Set<String>> lastInstantAndIncrPartitions = getIncrementalPartitions(type);
+        Option<HoodieInstant> lastCompleteTableServiceInstant = lastInstantAndIncrPartitions.getLeft();
+        Set<String> incrementalPartitions = lastInstantAndIncrPartitions.getRight();
+
+        if (lastCompleteTableServiceInstant.isPresent()) {
+          if (!incrementalPartitions.isEmpty()) {
+            LOG.info("Fetched incremental partitions for " + type + ". " + incrementalPartitions + ". Instant " + instantTime);
+            return new ArrayList<>(incrementalPartitions);
+          } else {
+            // handle the case the writer just commits the empty commits continuously
+            // the incremental partition list is empty we just skip the scheduling
+            LOG.info("Incremental partitions are empty. Skip current schedule " + instantTime);
+            return Collections.emptyList();
+          }
+        }
+        // Last complete table service commit maybe archived.
+        // fall back to get all partitions.
+        LOG.info("No previous completed table service instant, fall back to get all partitions");
+      } catch (Exception ex) {
+        LOG.warn("Failed to get incremental partitions", ex);
+      }
+    }
+
+    // get all partitions
+    LOG.info("Start to fetch all partitions for " + type + ". Instant " + instantTime);
+    return FSUtils.getAllPartitionPaths(context, table.getMetaClient().getStorage(),
+        config.getMetadataConfig(), table.getMetaClient().getBasePath());
+  }
+
+  public Pair<Option<HoodieInstant>, Set<String>> getIncrementalPartitions(TableServiceType type) {
+    Pair<Option<HoodieInstant>, List<String>> missingPair = fetchMissingPartitions(type);
+    Option<HoodieInstant> lastCompleteTableServiceInstant = missingPair.getLeft();
+    List<String> missingPartitions = missingPair.getRight();
+
+    String leftBoundary = lastCompleteTableServiceInstant.isPresent()
+        ? missingPair.getLeft().get().requestedTime() : HoodieTimeline.INIT_INSTANT_TS;
+    String rightBoundary = instantTime;
+    // compute [leftBoundary, rightBoundary) as time window
+    HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
+    Set<String> partitionsInCommitMeta = table.getActiveTimeline().filterCompletedInstants().getCommitsTimeline().getInstantsAsStream()
+        .filter(instant -> {
+          // ignore lastCompleteTableServiceInstant(left boundary) itself
+          return !(lastCompleteTableServiceInstant.isPresent() && instant.equals(lastCompleteTableServiceInstant.get()));
+        })
+        .filter(this::filterCommitByTableType).flatMap(instant -> {
+          try {
+            String completionTime = instant.getCompletionTime();
+            if (completionTime.compareTo(leftBoundary) >= 0 && completionTime.compareTo(rightBoundary) < 0) {
+              HoodieCommitMetadata metadata = TimelineUtils.getCommitMetadata(instant, activeTimeline);
+              return metadata.getWriteStats().stream().map(HoodieWriteStat::getPartitionPath);
+            }
+            return Stream.empty();
+          } catch (IOException e) {
+            throw new HoodieIOException("Failed to get commit meta " + instant, e);
+          }
+        }).collect(Collectors.toSet());
+
+    partitionsInCommitMeta.addAll(missingPartitions);
+    return Pair.of(lastCompleteTableServiceInstant, partitionsInCommitMeta);
+  }
+
+  private boolean filterCommitByTableType(HoodieInstant instant) {
+    switch (table.getMetaClient().getTableType()) {
+      case MERGE_ON_READ: {
+        // for mor only take care of delta commit and replace commit
+        return MOR_COMMITS.contains(instant.getAction());
+      }
+      case COPY_ON_WRITE: {
+        // for mor only take care of commit and replace commit
+        return COW_COMMITS.contains(instant.getAction());
+      }
+      default:
+        throw new HoodieException("Un-supported table type " + table.getMetaClient().getTableType());
+    }
+  }
+
+  public Pair<Option<HoodieInstant>, List<String>> fetchMissingPartitions(TableServiceType tableServiceType) {
+    if (!config.isIncrementalTableServiceEnabled()) {
+      return Pair.of(Option.empty(), Collections.emptyList());
+    }
+
+    Option<HoodieInstant> instant = Option.empty();
+    List<String> missingPartitions = new ArrayList<>();
+
+    switch (tableServiceType) {
+      case COMPACT:
+      case LOG_COMPACT: {
+        Option<HoodieInstant> lastCompactionCommitInstant = table.getActiveTimeline()
+            .filterCompletedInstants().getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION)).lastInstant();
+        if (lastCompactionCommitInstant.isPresent()) {
+          instant = lastCompactionCommitInstant;
+          String action = tableServiceType.equals(TableServiceType.COMPACT) ? HoodieTimeline.COMPACTION_ACTION : HoodieTimeline.LOG_COMPACTION_ACTION;
+          HoodieInstant compactionPlanInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, action,
+              instant.get().requestedTime(), InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+          Option<byte[]> details = table.getMetaClient().getActiveTimeline().readCompactionPlanAsBytes(compactionPlanInstant);
+          HoodieCompactionPlan compactionPlan = CompactionUtils.getCompactionPlan(table.getMetaClient(), details);
+          if (compactionPlan.getMissingSchedulePartitions() != null) {
+            missingPartitions = compactionPlan.getMissingSchedulePartitions();
+          }
+        }
+        break;
+      }
+      case CLUSTER: {
+        Option<HoodieInstant> lastClusteringInstant = table.getActiveTimeline().filterCompletedInstants().getLastClusteringInstant();
+        if (lastClusteringInstant.isPresent()) {
+          instant = lastClusteringInstant;
+          Option<Pair<HoodieInstant, HoodieClusteringPlan>> clusteringPlan = ClusteringUtils.getClusteringPlan(table.getMetaClient(), lastClusteringInstant.get());
+          if (clusteringPlan.isPresent() && clusteringPlan.get().getRight().getMissingSchedulePartitions() != null) {
+            missingPartitions = clusteringPlan.get().getRight().getMissingSchedulePartitions();
+          }
+        }
+        break;
+      }
+      default:
+        throw new HoodieException("Un-supported incremental table service " + tableServiceType);
+    }
+
+    return Pair.of(instant, missingPartitions);
+  }
+
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java
new file mode 100644
index 00000000000..717d508f7ae
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action;
+
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.List;
+
+/**
+ * Marking interface for table service strategy that utilize incremental partitions.
+ *
+ * <p> Any strategy class that implements this `IncrementalPartitionAwareStrategy` could have the ability to perform incremental partitions processing.
+ * Currently, Incremental partitions will be passed to the strategy instance as a best-effort. In the following cases, the partitions would fallback to full partition list:
+ *
+ * <ul>
+ *   <li> Executing Table Service for the first time. </li>
+ *   <li> The last completed table service instant is archived. </li>
+ *   <li> Any exception thrown during retrieval of incremental partitions. </li>
+ * </ul>
+ */
+public interface IncrementalPartitionAwareStrategy {
+  
+  /**
+   * Filter the given incremental partitions.
+   * @param writeConfig
+   * @param partitions
+   * @return Pair of final processing partition paths and filtered partitions which will be recorded as missing partitions.
+   * Different strategies can individually implement whether to record, or which partitions to record as missing partitions.
+   */
+  Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions);
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
index d698053d7b5..a92f8c87ba5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
@@ -21,6 +21,7 @@ package org.apache.hudi.table.action.cluster;
 import org.apache.hudi.avro.model.HoodieClusteringPlan;
 import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
 import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -31,17 +32,19 @@ import org.apache.hudi.common.util.ReflectionUtils;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.BaseActionExecutor;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
 import org.apache.hudi.table.action.cluster.strategy.ClusteringPlanStrategy;
+import org.apache.hudi.util.Lazy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
-public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, O, Option<HoodieClusteringPlan>> {
+public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseTableServicePlanActionExecutor<T, I, K, O, Option<HoodieClusteringPlan>> {
 
   private static final Logger LOG = LoggerFactory.getLogger(ClusteringPlanActionExecutor.class);
 
@@ -84,7 +87,9 @@ public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseActionExecutor
         ClusteringPlanStrategy.checkAndGetClusteringPlanStrategy(config),
             new Class<?>[] {HoodieTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, table, context, config);
 
-    return strategy.generateClusteringPlan();
+    Lazy<List<String>> partitions = Lazy.lazily(() -> getPartitions(strategy, TableServiceType.CLUSTER));
+
+    return strategy.generateClusteringPlan(this, partitions);
   }
 
   @Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
index ecc3706f674..47c483e265b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
@@ -18,12 +18,14 @@
 
 package org.apache.hudi.table.action.cluster;
 
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieClusteringException;
 
 import org.joda.time.DateTime;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -42,11 +44,11 @@ import java.util.stream.Stream;
  */
 public class ClusteringPlanPartitionFilter {
 
-  public static List<String> filter(List<String> partitions, HoodieWriteConfig config) {
+  public static Pair<List<String>, List<String>> filter(List<String> partitions, HoodieWriteConfig config) {
     ClusteringPlanPartitionFilterMode mode = config.getClusteringPlanPartitionFilterMode();
     switch (mode) {
       case NONE:
-        return partitions;
+        return Pair.of(partitions, Collections.emptyList());
       case RECENT_DAYS:
         return recentDaysFilter(partitions, config);
       case SELECTED_PARTITIONS:
@@ -58,7 +60,7 @@ public class ClusteringPlanPartitionFilter {
     }
   }
 
-  private static List<String> dayRollingFilter(List<String> partitions, HoodieWriteConfig config) {
+  private static Pair<List<String>, List<String>> dayRollingFilter(List<String> partitions, HoodieWriteConfig config) {
     int hour = DateTime.now().getHourOfDay();
     int len = partitions.size();
     List<String> selectPt = new ArrayList<>();
@@ -68,20 +70,22 @@ public class ClusteringPlanPartitionFilter {
         selectPt.add(partitions.get(i));
       }
     }
-    return selectPt;
+    return Pair.of(selectPt, Collections.emptyList());
   }
 
-  private static List<String> recentDaysFilter(List<String> partitions, HoodieWriteConfig config) {
+  private static Pair<List<String>, List<String>> recentDaysFilter(List<String> partitions, HoodieWriteConfig config) {
     int targetPartitionsForClustering = config.getTargetPartitionsForClustering();
     int skipPartitionsFromLatestForClustering = config.getSkipPartitionsFromLatestForClustering();
-    return partitions.stream()
-        .sorted(Comparator.reverseOrder())
-        .skip(Math.max(skipPartitionsFromLatestForClustering, 0))
+    int skipOffset = Math.max(skipPartitionsFromLatestForClustering, 0);
+    List<String> sortedPartitions = partitions.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList());
+    List<String> missingPartitions = sortedPartitions.subList(0, skipOffset);
+    return Pair.of(sortedPartitions.stream()
+        .skip(skipOffset)
         .limit(targetPartitionsForClustering > 0 ? targetPartitionsForClustering : partitions.size())
-        .collect(Collectors.toList());
+        .collect(Collectors.toList()), missingPartitions);
   }
 
-  private static List<String> selectedPartitionsFilter(List<String> partitions, HoodieWriteConfig config) {
+  private static Pair<List<String>, List<String>> selectedPartitionsFilter(List<String> partitions, HoodieWriteConfig config) {
     Stream<String> filteredPartitions = partitions.stream();
 
     String beginPartition = config.getBeginPartitionForClustering();
@@ -94,6 +98,6 @@ public class ClusteringPlanPartitionFilter {
       filteredPartitions = filteredPartitions.filter(path -> path.compareTo(endPartition) <= 0);
     }
 
-    return filteredPartitions.collect(Collectors.toList());
+    return Pair.of(filteredPartitions.collect(Collectors.toList()), Collections.emptyList());
   }
 }
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
index 27fea59fa9f..095066d73e7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.table.view.TableFileSystemView;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.common.util.collection.Triple;
 import org.apache.hudi.config.HoodieClusteringConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
@@ -95,7 +96,7 @@ public abstract class BaseConsistentHashingBucketClusteringPlanStrategy<T extend
    * Generate cluster group based on split, merge and sort rules
    */
   @Override
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     Option<HoodieConsistentHashingMetadata> metadata = ConsistentBucketIndexUtils.loadMetadata(getHoodieTable(), partitionPath);
     ValidationUtils.checkArgument(metadata.isPresent(), "Metadata is empty for partition: " + partitionPath);
     ConsistentBucketIdentifier identifier = new ConsistentBucketIdentifier(metadata.get());
@@ -128,7 +129,7 @@ public abstract class BaseConsistentHashingBucketClusteringPlanStrategy<T extend
             .build();
       }).collect(Collectors.toList()));
     }
-    return ret.stream();
+    return Pair.of(ret.stream(), true);
   }
 
   /**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
index a6894388f6d..084beee85f1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
@@ -32,7 +32,9 @@ import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieClusteringConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
 import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
+import org.apache.hudi.util.Lazy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -101,7 +103,7 @@ public abstract class ClusteringPlanStrategy<T,I,K,O> implements Serializable {
    *
    * If there is no data available to cluster, return None.
    */
-  public abstract Option<HoodieClusteringPlan> generateClusteringPlan();
+  public abstract Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions);
 
   /**
    * Check if the clustering can proceed. If not (i.e., return false), the PlanStrategy will generate an empty plan to stop the scheduling.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
index 515d6532871..3f0a97eff22 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -22,22 +22,27 @@ import org.apache.hudi.avro.model.HoodieClusteringGroup;
 import org.apache.hudi.avro.model.HoodieClusteringPlan;
 import org.apache.hudi.avro.model.HoodieClusteringStrategy;
 import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
 import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilter;
+import org.apache.hudi.util.Lazy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -45,7 +50,7 @@ import java.util.stream.Stream;
 /**
  * Scheduling strategy with restriction that clustering groups can only contain files from same partition.
  */
-public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends ClusteringPlanStrategy<T,I,K,O> {
+public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends ClusteringPlanStrategy<T,I,K,O> implements IncrementalPartitionAwareStrategy {
   private static final Logger LOG = LoggerFactory.getLogger(PartitionAwareClusteringPlanStrategy.class);
 
   public PartitionAwareClusteringPlanStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
@@ -54,8 +59,10 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
 
   /**
    * Create Clustering group based on files eligible for clustering in the partition.
+   * return stream of HoodieClusteringGroup and boolean partial Scheduled indicating whether all given fileSlices in the current partition have been processed.
+   * For example, if some file slices will not be processed due to writeConfig.getClusteringMaxNumGroups(), then return false
    */
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     HoodieWriteConfig writeConfig = getWriteConfig();
 
     List<Pair<List<FileSlice>, Integer>> fileSliceGroups = new ArrayList<>();
@@ -68,6 +75,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
             - (o1.getBaseFile().isPresent() ? o1.getBaseFile().get().getFileSize() : writeConfig.getParquetMaxFileSize())));
 
     long totalSizeSoFar = 0;
+    boolean partialScheduled = false;
 
     for (FileSlice currentSlice : sortedFileSlices) {
       long currentSize = currentSlice.getBaseFile().isPresent() ? currentSlice.getBaseFile().get().getFileSize() : writeConfig.getParquetMaxFileSize();
@@ -83,6 +91,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
         // if fileSliceGroups's size reach the max group, stop loop
         if (fileSliceGroups.size() >= writeConfig.getClusteringMaxNumGroups()) {
           LOG.info("Having generated the maximum number of groups : " + writeConfig.getClusteringMaxNumGroups());
+          partialScheduled = true;
           break;
         }
       }
@@ -102,25 +111,23 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
       }
     }
 
-    return fileSliceGroups.stream().map(fileSliceGroup ->
+    return Pair.of(fileSliceGroups.stream().map(fileSliceGroup ->
         HoodieClusteringGroup.newBuilder()
             .setSlices(getFileSliceInfo(fileSliceGroup.getLeft()))
             .setNumOutputFileGroups(fileSliceGroup.getRight())
             .setMetrics(buildMetrics(fileSliceGroup.getLeft()))
-            .build());
+            .build()), partialScheduled);
   }
 
   /**
    * Return list of partition paths to be considered for clustering.
    */
-  protected List<String> filterPartitionPaths(List<String> partitionPaths) {
-    List<String> filteredPartitions = ClusteringPlanPartitionFilter.filter(partitionPaths, getWriteConfig());
-    LOG.debug("Filtered to the following partitions: " + filteredPartitions);
-    return filteredPartitions;
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions) {
+    return ClusteringPlanPartitionFilter.filter(partitions, getWriteConfig());
   }
 
   @Override
-  public Option<HoodieClusteringPlan> generateClusteringPlan() {
+  public Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions) {
     if (!checkPrecondition()) {
       return Option.empty();
     }
@@ -131,36 +138,74 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
 
     String partitionSelected = config.getClusteringPartitionSelected();
     LOG.info("Scheduling clustering partitionSelected: {}", partitionSelected);
-    List<String> partitionPaths;
+    List<String> partitionPaths = new ArrayList<>();
+    List<String> missingPartitions = new ArrayList<>();
+    List<HoodieClusteringGroup> clusteringGroups;
+    int clusteringMaxNumGroups = getWriteConfig().getClusteringMaxNumGroups();
 
     if (StringUtils.isNullOrEmpty(partitionSelected)) {
       // get matched partitions if set
-      partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(
-          getEngineContext(), metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath()));
+      partitionPaths = getRegexPatternMatchedPartitions(config, partitions.get());
       // filter the partition paths if needed to reduce list status
     } else {
       partitionPaths = Arrays.asList(partitionSelected.split(","));
+      // Users may temporarily set specific partitions for clustering.
+      // Ensure the coherence of the missing partitions.
+      missingPartitions = (List<String>)executor.fetchMissingPartitions(TableServiceType.CLUSTER).getRight();
     }
 
-    partitionPaths = filterPartitionPaths(partitionPaths);
+    Pair<List<String>, List<String>> partitionsPair = filterPartitionPaths(getWriteConfig(), partitionPaths);
+    partitionPaths = partitionsPair.getLeft();
+    missingPartitions.addAll(partitionsPair.getRight());
     LOG.info("Scheduling clustering partitionPaths: {}", partitionPaths);
+    LOG.info("Missing Scheduled clustering partitionPaths: {}", missingPartitions);
 
     if (partitionPaths.isEmpty()) {
       // In case no partitions could be picked, return no clustering plan
       return Option.empty();
     }
 
-    List<HoodieClusteringGroup> clusteringGroups = getEngineContext()
-        .flatMap(
-            partitionPaths,
-            partitionPath -> {
-              List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
-              return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).limit(getWriteConfig().getClusteringMaxNumGroups());
-            },
-            partitionPaths.size())
-        .stream()
-        .limit(getWriteConfig().getClusteringMaxNumGroups())
-        .collect(Collectors.toList());
+    List<Pair<List<HoodieClusteringGroup>, String>> res = getEngineContext().map(partitionPaths, partitionPath -> {
+      List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
+      Pair<Stream<HoodieClusteringGroup>, Boolean> groupPair = buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible);
+      List<HoodieClusteringGroup> clusteringGroupsPartition = groupPair.getLeft().collect(Collectors.toList());
+      boolean partialScheduled = groupPair.getRight();
+      // return missed partition path
+      // because the candidate fileSlices in the current partition have not been completely processed.
+      if (clusteringGroupsPartition.size() > clusteringMaxNumGroups) {
+        return Pair.of(clusteringGroupsPartition.subList(0, clusteringMaxNumGroups), partitionPath);
+      } else if (partialScheduled) {
+        return Pair.of(clusteringGroupsPartition, partitionPath);
+      } else {
+        return Pair.of(clusteringGroupsPartition, "");
+      }
+    }, partitionPaths.size());
+
+    if (config.isIncrementalTableServiceEnabled()) {
+      Set<String> skippedPartitions = new HashSet<>();
+      List<HoodieClusteringGroup> collectedGroups = res.stream().flatMap(pair -> {
+        String missingPartition = pair.getRight();
+        if (!StringUtils.isNullOrEmpty(missingPartition)) {
+          // missingPartition value is not empty, which means it related candidate fileSlices all not all processed.
+          // so that we need to mark this kind of partition as missing partition.
+          skippedPartitions.add(missingPartition);
+        }
+        return pair.getLeft().stream();
+      }).collect(Collectors.toList());
+
+      clusteringGroups = collectedGroups.stream().limit(clusteringMaxNumGroups).collect(Collectors.toList());
+      // sublist of collectedGroups are skipped, marking related partition as missing partitions.
+      collectedGroups.subList(Math.min(clusteringMaxNumGroups, collectedGroups.size()), collectedGroups.size()).forEach(group -> {
+        String missed = group.getSlices().get(0).getPartitionPath();
+        skippedPartitions.add(missed);
+      });
+
+      skippedPartitions.addAll(missingPartitions);
+      missingPartitions = new ArrayList<>(skippedPartitions);
+    } else {
+      clusteringGroups = res.stream().flatMap(pair -> pair.getLeft().stream()).limit(clusteringMaxNumGroups).collect(Collectors.toList());
+      missingPartitions = null;
+    }
 
     if (clusteringGroups.isEmpty()) {
       LOG.warn("No data available to cluster");
@@ -178,6 +223,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
         .setExtraMetadata(getExtraMetadata())
         .setVersion(getPlanVersion())
         .setPreserveHoodieMetadata(true)
+        .setMissingSchedulePartitions(missingPartitions)
         .build());
   }
 
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
index e83800e45da..bd97d40f7fc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
@@ -34,7 +34,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieCompactionException;
 import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.BaseActionExecutor;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
 import org.apache.hudi.table.action.compact.plan.generators.BaseHoodieCompactionPlanGenerator;
 import org.apache.hudi.table.action.compact.plan.generators.HoodieCompactionPlanGenerator;
 import org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator;
@@ -50,7 +50,7 @@ import java.util.Map;
 import static org.apache.hudi.common.util.CollectionUtils.nonEmpty;
 import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
 
-public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, O, Option<HoodieCompactionPlan>> {
+public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseTableServicePlanActionExecutor<T, I, K, O, Option<HoodieCompactionPlan>> {
 
   private static final Logger LOG = LoggerFactory.getLogger(ScheduleCompactionActionExecutor.class);
   private WriteOperationType operationType;
@@ -73,9 +73,9 @@ public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseActionExec
 
   private void initPlanGenerator(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table) {
     if (WriteOperationType.COMPACT.equals(operationType)) {
-      planGenerator = new HoodieCompactionPlanGenerator(table, context, config);
+      planGenerator = new HoodieCompactionPlanGenerator(table, context, config, this);
     } else {
-      planGenerator = new HoodieLogCompactionPlanGenerator(table, context, config);
+      planGenerator = new HoodieLogCompactionPlanGenerator(table, context, config, this);
     }
   }
 
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
index 1884f4c5d40..0d392bc5ff9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
@@ -22,7 +22,6 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
 import org.apache.hudi.common.data.HoodieAccumulator;
 import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.CompactionOperation;
 import org.apache.hudi.common.model.FileSlice;
 import org.apache.hudi.common.model.HoodieBaseFile;
@@ -41,6 +40,7 @@ import org.apache.hudi.common.util.ValidationUtils;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
 import org.apache.hudi.table.action.compact.CompactHelpers;
 
 import org.slf4j.Logger;
@@ -64,11 +64,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
   protected final HoodieTable<T, I, K, O> hoodieTable;
   protected final HoodieWriteConfig writeConfig;
   protected final transient HoodieEngineContext engineContext;
+  protected final BaseTableServicePlanActionExecutor executor;
 
-  public BaseHoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
+  public BaseHoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+                                           BaseTableServicePlanActionExecutor executor) {
     this.hoodieTable = table;
     this.writeConfig = writeConfig;
     this.engineContext = engineContext;
+    this.executor = executor;
   }
 
   @Nullable
@@ -82,13 +85,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
     // TODO - rollback any compactions in flight
     HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
     CompletionTimeQueryView completionTimeQueryView = metaClient.getTimelineLayout().getTimelineFactory().createCompletionTimeQueryView(metaClient);
-    List<String> partitionPaths = FSUtils.getAllPartitionPaths(
-        engineContext, metaClient.getStorage(), writeConfig.getMetadataConfig(), metaClient.getBasePath());
+    List<String> partitionPaths = getPartitions();
 
     int allPartitionSize = partitionPaths.size();
 
     // filter the partition paths if needed to reduce list status
-    partitionPaths = filterPartitionPathsByStrategy(partitionPaths);
+    Pair<List<String>, List<String>> partitionPair = filterPartitionPathsByStrategy(partitionPaths);
+    partitionPaths = partitionPair.getLeft();
+
     LOG.info("Strategy: {} matched {} partition paths from all {} partitions for table {}",
         writeConfig.getCompactionStrategy().getClass().getSimpleName(), partitionPaths.size(), allPartitionSize,
         hoodieTable.getConfig().getBasePath());
@@ -169,7 +173,7 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
     }
 
     // Filter the compactions with the passed in filter. This lets us choose most effective compactions only
-    HoodieCompactionPlan compactionPlan = getCompactionPlan(metaClient, operations);
+    HoodieCompactionPlan compactionPlan = getCompactionPlan(metaClient, operations, partitionPair);
     ValidationUtils.checkArgument(
         compactionPlan.getOperations().stream().noneMatch(
             op -> fgIdsInPendingCompactionAndClustering.contains(new HoodieFileGroupId(op.getPartitionPath(), op.getFileId()))),
@@ -182,12 +186,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
     return compactionPlan;
   }
 
-  protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations);
+  protected abstract List<String> getPartitions();
+
+  protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>,List<String>> partitionPair);
 
   protected abstract boolean filterLogCompactionOperations();
 
-  protected List<String> filterPartitionPathsByStrategy(List<String> partitionPaths) {
-    return partitionPaths;
+  protected Pair<List<String>, List<String>> filterPartitionPathsByStrategy(List<String> partitionPaths) {
+    return Pair.of(partitionPaths, Collections.emptyList());
   }
 
   protected boolean filterFileSlice(FileSlice fileSlice, String lastCompletedInstantTime,
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
index a93ece710b0..378461f1b98 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
@@ -22,11 +22,13 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
 import org.apache.hudi.common.engine.HoodieEngineContext;
 import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.util.CompactionUtils;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
 import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
 
 import org.slf4j.Logger;
@@ -43,22 +45,28 @@ public class HoodieCompactionPlanGenerator<T extends HoodieRecordPayload, I, K,
 
   private final CompactionStrategy compactionStrategy;
 
-  public HoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
-    super(table, engineContext, writeConfig);
+  public HoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+                                       BaseTableServicePlanActionExecutor executor) {
+    super(table, engineContext, writeConfig, executor);
     this.compactionStrategy = writeConfig.getCompactionStrategy();
     LOG.info("Compaction Strategy used is: " + compactionStrategy.toString());
   }
 
   @Override
-  protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations) {
+  protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>, List<String>> partitionPair) {
     // Filter the compactions with the passed in filter. This lets us choose most effective
     // compactions only
     return compactionStrategy.generateCompactionPlan(writeConfig, operations,
-        CompactionUtils.getAllPendingCompactionPlans(metaClient).stream().map(Pair::getValue).collect(toList()));
+        CompactionUtils.getAllPendingCompactionPlans(metaClient).stream().map(Pair::getValue).collect(toList()), getStrategyParams(), partitionPair);
   }
 
   @Override
-  protected List<String> filterPartitionPathsByStrategy(List<String> partitionPaths) {
+  protected List<String> getPartitions() {
+    return executor.getPartitions(compactionStrategy, TableServiceType.COMPACT);
+  }
+
+  @Override
+  protected Pair<List<String>, List<String>> filterPartitionPathsByStrategy(List<String> partitionPaths) {
     return compactionStrategy.filterPartitionPaths(writeConfig, partitionPaths);
   }
 
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
index a81ee663fa9..5d9337ac4e7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
@@ -26,13 +26,16 @@ import org.apache.hudi.common.model.FileSlice;
 import org.apache.hudi.common.model.HoodieFileGroupId;
 import org.apache.hudi.common.model.HoodieLogFile;
 import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner;
 import org.apache.hudi.common.table.log.InstantRange;
 import org.apache.hudi.common.util.CompactionUtils;
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
 import org.apache.hudi.table.action.compact.LogCompactionExecutionHelper;
 
 import org.slf4j.Logger;
@@ -45,25 +48,33 @@ import java.util.stream.Collectors;
 public class HoodieLogCompactionPlanGenerator<T extends HoodieRecordPayload, I, K, O> extends BaseHoodieCompactionPlanGenerator<T, I, K, O> {
 
   private static final Logger LOG = LoggerFactory.getLogger(HoodieLogCompactionPlanGenerator.class);
+  private final HoodieCompactionStrategy compactionStrategy;
 
-  public HoodieLogCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
-    super(table, engineContext, writeConfig);
-  }
-
-  @Override
-  protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations) {
-    HoodieCompactionStrategy compactionStrategy = HoodieCompactionStrategy.newBuilder()
+  public HoodieLogCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+                                          BaseTableServicePlanActionExecutor executor) {
+    super(table, engineContext, writeConfig, executor);
+    this.compactionStrategy = HoodieCompactionStrategy.newBuilder()
         .setStrategyParams(getStrategyParams())
         .setCompactorClassName(LogCompactionExecutionHelper.class.getName())
         .build();
+  }
+
+  @Override
+  protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>, List<String>> partitionPair) {
     return HoodieCompactionPlan.newBuilder()
         .setOperations(operations)
         .setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION)
         .setStrategy(compactionStrategy)
+        .setMissingSchedulePartitions(partitionPair.getRight())
         .setPreserveHoodieMetadata(true)
         .build();
   }
 
+  @Override
+  protected List<String> getPartitions() {
+    return executor.getPartitions(compactionStrategy, TableServiceType.LOG_COMPACT);
+  }
+
   @Override
   protected boolean filterFileSlice(FileSlice fileSlice, String lastCompletedInstantTime,
                                     Set<HoodieFileGroupId> pendingFileGroupIds, Option<InstantRange> instantRange) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
index d93a50fd673..c2bd139fe9e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
@@ -20,9 +20,11 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -34,8 +36,10 @@ import java.util.List;
 public class BoundedIOCompactionStrategy extends CompactionStrategy {
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
-      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+                                                                            List<HoodieCompactionOperation> operations,
+                                                                            List<HoodieCompactionPlan> pendingCompactionPlans) {
+    ArrayList<String> missingPartitions = new ArrayList<>();
     // Iterate through the operations in order and accept operations as long as we are within the
     // IO limit
     // Preserves the original ordering of compactions
@@ -43,12 +47,15 @@ public class BoundedIOCompactionStrategy extends CompactionStrategy {
     long targetIORemaining = writeConfig.getTargetIOPerCompactionInMB();
     for (HoodieCompactionOperation op : operations) {
       long opIo = op.getMetrics().get(TOTAL_IO_MB).longValue();
-      targetIORemaining -= opIo;
-      finalOperations.add(op);
-      if (targetIORemaining <= 0) {
-        return finalOperations;
+      if (targetIORemaining > 0) {
+        targetIORemaining -= opIo;
+        finalOperations.add(op);
+      } else if (writeConfig.isIncrementalTableServiceEnabled()) {
+        missingPartitions.add(op.getPartitionPath());
+      } else  {
+        return Pair.of(finalOperations, Collections.emptyList());
       }
     }
-    return finalOperations;
+    return Pair.of(finalOperations, missingPartitions);
   }
 }
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
index 09c19b1aabe..3944a7658a2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
@@ -20,10 +20,12 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
@@ -45,28 +47,33 @@ public class BoundedPartitionAwareCompactionStrategy extends DayBasedCompactionS
       ThreadLocal.withInitial(() -> new SimpleDateFormat(DayBasedCompactionStrategy.DATE_PARTITION_FORMAT));
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
       List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
     // The earliest partition to compact - current day minus the target partitions limit
     String earliestPartitionPathToCompact =
         dateFormat.get().format(getDateAtOffsetFromToday(-1 * writeConfig.getTargetPartitionsPerDayBasedCompaction()));
     // Filter out all partitions greater than earliestPartitionPathToCompact
-
-    return operations.stream().collect(Collectors.groupingBy(HoodieCompactionOperation::getPartitionPath)).entrySet()
+    List<HoodieCompactionOperation> res = operations.stream().collect(Collectors.groupingBy(HoodieCompactionOperation::getPartitionPath)).entrySet()
         .stream().sorted(Map.Entry.comparingByKey(DayBasedCompactionStrategy.comparator))
         .filter(e -> DayBasedCompactionStrategy.comparator.compare(earliestPartitionPathToCompact, e.getKey()) >= 0)
         .flatMap(e -> e.getValue().stream()).collect(Collectors.toList());
+
+    return Pair.of(res, Collections.emptyList());
   }
 
+  /**
+   * For BoundedPartitionAwareCompactionStrategy, we will not record rolled partitions as missing partitions.
+   */
   @Override
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
     // The earliest partition to compact - current day minus the target partitions limit
     String earliestPartitionPathToCompact =
         dateFormat.get().format(getDateAtOffsetFromToday(-1 * writeConfig.getTargetPartitionsPerDayBasedCompaction()));
     // Get all partitions and sort them
-    return partitionPaths.stream().map(partition -> partition.replace("/", "-"))
+    List<String> res = partitionPaths.stream().map(partition -> partition.replace("/", "-"))
         .sorted(Comparator.reverseOrder()).map(partitionPath -> partitionPath.replace("-", "/"))
         .filter(e -> DayBasedCompactionStrategy.comparator.compare(earliestPartitionPathToCompact, e) >= 0).collect(Collectors.toList());
+    return Pair.of(res, Collections.emptyList());
   }
 
   public static Date getDateAtOffsetFromToday(int offset) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
index 5b2dba7ab7a..718dc685749 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
@@ -23,13 +23,16 @@ import org.apache.hudi.avro.model.HoodieCompactionPlan;
 import org.apache.hudi.client.utils.FileSliceMetricUtils;
 import org.apache.hudi.common.model.FileSlice;
 import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
 
 import java.io.Serializable;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Strategy for compaction. Pluggable implementation to define how compaction should be done. The over-ridden
@@ -37,7 +40,7 @@ import java.util.Map;
  * compaction operation to run in a single compaction. Implementation of CompactionStrategy cannot hold any state.
  * Difference instantiations can be passed in every time
  */
-public abstract class CompactionStrategy implements Serializable {
+public abstract class CompactionStrategy implements IncrementalPartitionAwareStrategy, Serializable {
 
   public static final String TOTAL_IO_READ_MB = "TOTAL_IO_READ_MB";
   public static final String TOTAL_IO_WRITE_MB = "TOTAL_IO_WRITE_MB";
@@ -71,11 +74,18 @@ public abstract class CompactionStrategy implements Serializable {
    * @return Compaction plan to be scheduled.
    */
   public HoodieCompactionPlan generateCompactionPlan(HoodieWriteConfig writeConfig,
-      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans, Map<String, String> params, Pair<List<String>, List<String>> partitionPair) {
     // Strategy implementation can overload this method to set specific compactor-id
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = orderAndFilter(writeConfig, operations, pendingCompactionPlans);
+    List<HoodieCompactionOperation> operationsToProcess = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    missingPartitions.addAll(partitionPair.getRight());
+    List<String> res = writeConfig.isIncrementalTableServiceEnabled() ? missingPartitions.stream().distinct().collect(Collectors.toList()) : null;
     return HoodieCompactionPlan.newBuilder()
-        .setOperations(orderAndFilter(writeConfig, operations, pendingCompactionPlans))
-        .setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION).build();
+        .setOperations(operationsToProcess)
+        .setMissingSchedulePartitions(res)
+        .setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION)
+        .build();
   }
 
   /**
@@ -85,11 +95,12 @@ public abstract class CompactionStrategy implements Serializable {
    * @param writeConfig config for this compaction is passed in
    * @param operations list of compactions collected
    * @param pendingCompactionPlans Pending Compaction Plans for strategy to schedule next compaction plan
-   * @return list of compactions to perform in this run
+   * @return Pair of list of compactions to perform in this run and missing partitions.
+   *         Filtered out operations related partitions are missing partitions.
    */
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
       List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
-    return operations;
+    return Pair.of(operations, Collections.emptyList());
   }
 
   /**
@@ -99,7 +110,7 @@ public abstract class CompactionStrategy implements Serializable {
    * @param allPartitionPaths
    * @return
    */
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
-    return allPartitionPaths;
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+    return Pair.of(allPartitionPaths, Collections.emptyList());
   }
 }
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
index da90269509d..882f880069e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
@@ -21,8 +21,11 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -39,21 +42,30 @@ public class CompositeCompactionStrategy extends CompactionStrategy {
   }
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig, List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+                                                                            List<HoodieCompactionOperation> operations,
+                                                                            List<HoodieCompactionPlan> pendingCompactionPlans) {
     List<HoodieCompactionOperation> finalOperations = operations;
+    List<String> missingPartitions = new ArrayList<>();
     for (CompactionStrategy strategy : strategies) {
-      finalOperations = strategy.orderAndFilter(writeConfig, finalOperations, pendingCompactionPlans);
+      Pair<List<HoodieCompactionOperation>, List<String>> resPair = strategy.orderAndFilter(writeConfig, finalOperations, pendingCompactionPlans);
+      finalOperations = resPair.getLeft();
+      missingPartitions.addAll(resPair.getRight());
     }
-    return finalOperations;
+    return Pair.of(finalOperations, missingPartitions);
   }
 
   @Override
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
-    List<String> finalPartitionPaths = allPartitionPaths;
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+    List<String> partitionsToProcess = allPartitionPaths;
+    List<String> missingPartitions = new ArrayList<>();
+
     for (CompactionStrategy strategy : strategies) {
-      finalPartitionPaths = strategy.filterPartitionPaths(writeConfig, finalPartitionPaths);
+      Pair<List<String>, List<String>> innerRes = strategy.filterPartitionPaths(writeConfig, partitionsToProcess);
+      partitionsToProcess = innerRes.getLeft();
+      missingPartitions.addAll(innerRes.getRight());
     }
-    return finalPartitionPaths;
+    return writeConfig.isIncrementalTableServiceEnabled() ? Pair.of(partitionsToProcess, missingPartitions) : Pair.of(partitionsToProcess, Collections.emptyList());
   }
 
   @Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
index d77d22d05c5..c7344c4590b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
@@ -18,11 +18,13 @@
 
 package org.apache.hudi.table.action.compact.strategy;
 
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieException;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
@@ -60,11 +62,15 @@ public class DayBasedCompactionStrategy extends BoundedIOCompactionStrategy {
     return comparator;
   }
 
+  /**
+   * For DayBasedCompactionStrategy, we will not record rolled partitions as missing partitions.
+   */
   @Override
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
-    return allPartitionPaths.stream().sorted(comparator)
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+    List<String> res = allPartitionPaths.stream().sorted(comparator)
         .collect(Collectors.toList()).subList(0, Math.min(allPartitionPaths.size(),
             writeConfig.getTargetPartitionsPerDayBasedCompaction()));
+    return Pair.of(res, Collections.emptyList());
   }
 
   /**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
index 6f79b684d0a..efacfe8b650 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -34,12 +36,30 @@ public class LogFileNumBasedCompactionStrategy extends BoundedIOCompactionStrate
     implements Comparator<HoodieCompactionOperation> {
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig, List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+                                                                            List<HoodieCompactionOperation> operations,
+                                                                            List<HoodieCompactionPlan> pendingCompactionPlans) {
     Long numThreshold = writeConfig.getCompactionLogFileNumThreshold();
+    ArrayList<String> missingPartitions = new ArrayList<>();
+    boolean incrementalTableServiceEnabled = writeConfig.isIncrementalTableServiceEnabled();
     List<HoodieCompactionOperation> filterOperator = operations.stream()
-        .filter(e -> e.getDeltaFilePaths().size() >= numThreshold)
+        .filter(e -> {
+          if (incrementalTableServiceEnabled && e.getDeltaFilePaths().size() < numThreshold) {
+            missingPartitions.add(e.getPartitionPath());
+          }
+          return e.getDeltaFilePaths().size() >= numThreshold;
+        })
         .sorted(this).collect(Collectors.toList());
-    return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+
+    if (incrementalTableServiceEnabled) {
+      Pair<List<HoodieCompactionOperation>, List<String>> resPair = super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+      List<HoodieCompactionOperation> compactOperations = resPair.getLeft();
+      List<String> innerMissingPartitions = resPair.getRight();
+      missingPartitions.addAll(innerMissingPartitions);
+      return Pair.of(compactOperations, missingPartitions);
+    } else {
+      return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+    }
   }
 
   @Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
index c165141dfc5..bd46569d827 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -38,15 +40,30 @@ public class LogFileSizeBasedCompactionStrategy extends BoundedIOCompactionStrat
     implements Comparator<HoodieCompactionOperation> {
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
-      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+                                                                            List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
     // Filter the file group which log files size is greater than the threshold in bytes.
     // Order the operations based on the reverse size of the logs and limit them by the IO
     long threshold = writeConfig.getCompactionLogFileSizeThreshold();
-    return super.orderAndFilter(writeConfig, operations.stream()
-            .filter(e -> e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) >= threshold)
-            .sorted(this).collect(Collectors.toList()),
-        pendingCompactionPlans);
+    ArrayList<String> missingPartitions = new ArrayList<>();
+    boolean incrementalTableServiceEnabled = writeConfig.isIncrementalTableServiceEnabled();
+    List<HoodieCompactionOperation> filterOperator = operations.stream()
+        .filter(e -> {
+          if (incrementalTableServiceEnabled && e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) < threshold) {
+            missingPartitions.add(e.getPartitionPath());
+          }
+          return e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) >= threshold;
+        }).sorted(this).collect(Collectors.toList());
+
+    if (incrementalTableServiceEnabled) {
+      Pair<List<HoodieCompactionOperation>, List<String>> resPair = super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+      List<HoodieCompactionOperation> compactOperations = resPair.getLeft();
+      List<String> innerMissingPartitions = resPair.getRight();
+      missingPartitions.addAll(innerMissingPartitions);
+      return Pair.of(compactOperations, missingPartitions);
+    } else {
+      return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+    }
   }
 
   @Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
index eb3b42ee7ca..a9f49fd1fc4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
@@ -18,18 +18,23 @@
 
 package org.apache.hudi.table.action.compact.strategy;
 
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 public class PartitionRegexBasedCompactionStrategy extends CompactionStrategy {
 
+  /**
+   * Regex filtered partitions are not included in missing partitions.
+   */
   @Override
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
     String regex = writeConfig.getCompactionSpecifyPartitionPathRegex();
     Pattern pattern = Pattern.compile(regex);
-    return allPartitionPaths.stream().filter(pattern.asPredicate()).collect(Collectors.toList());
+    return Pair.of(allPartitionPaths.stream().filter(pattern.asPredicate()).collect(Collectors.toList()), Collections.emptyList());
   }
 }
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
index ffc437bcd4c..d87354e42a3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -33,8 +35,9 @@ import java.util.List;
 public class UnBoundedCompactionStrategy extends CompactionStrategy {
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig config,
-      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionWorkloads) {
-    return operations;
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig config,
+                                                                            List<HoodieCompactionOperation> operations,
+                                                                            List<HoodieCompactionPlan> pendingCompactionWorkloads) {
+    return Pair.of(operations, Collections.emptyList());
   }
 }
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
index 66b5612e1ac..56ef8306c17 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
@@ -20,9 +20,11 @@ package org.apache.hudi.table.action.compact.strategy;
 
 import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -39,27 +41,27 @@ import java.util.stream.Collectors;
 public class UnBoundedPartitionAwareCompactionStrategy extends CompactionStrategy {
 
   @Override
-  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig config,
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig config,
       final List<HoodieCompactionOperation> operations, final List<HoodieCompactionPlan> pendingCompactionWorkloads) {
     BoundedPartitionAwareCompactionStrategy boundedPartitionAwareCompactionStrategy =
         new BoundedPartitionAwareCompactionStrategy();
     List<HoodieCompactionOperation> operationsToExclude =
-        boundedPartitionAwareCompactionStrategy.orderAndFilter(config, operations, pendingCompactionWorkloads);
+        boundedPartitionAwareCompactionStrategy.orderAndFilter(config, operations, pendingCompactionWorkloads).getLeft();
     List<HoodieCompactionOperation> allOperations = new ArrayList<>(operations);
     allOperations.removeAll(operationsToExclude);
-    return allOperations;
+    return Pair.of(allOperations, Collections.emptyList());
   }
 
   @Override
-  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
+  public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
     List<String> allPartitionPaths =
         partitionPaths.stream().map(partition -> partition.replace("/", "-")).sorted(Comparator.reverseOrder())
             .map(partitionPath -> partitionPath.replace("-", "/")).collect(Collectors.toList());
     BoundedPartitionAwareCompactionStrategy boundedPartitionAwareCompactionStrategy =
         new BoundedPartitionAwareCompactionStrategy();
     List<String> partitionsToExclude =
-        boundedPartitionAwareCompactionStrategy.filterPartitionPaths(writeConfig, partitionPaths);
+        boundedPartitionAwareCompactionStrategy.filterPartitionPaths(writeConfig, partitionPaths).getLeft();
     allPartitionPaths.removeAll(partitionsToExclude);
-    return allPartitionPaths;
+    return Pair.of(allPartitionPaths, Collections.emptyList());
   }
 }
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java
new file mode 100644
index 00000000000..84e21e821b4
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table;
+
+import org.apache.hudi.HoodieTestCommitGenerator;
+import org.apache.hudi.avro.model.HoodieClusteringGroup;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieClusteringStrategy;
+import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
+import org.apache.hudi.avro.model.HoodieSliceInfo;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.engine.TaskContextSupplier;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.testutils.FileCreateUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestBaseTableServicePlanActionExecutor {
+
+  @TempDir
+  File tempFile;
+  private static final StorageConfiguration<?> CONF = getDefaultStorageConf();
+  private final HoodieEngineContext context = new HoodieLocalEngineContext(CONF);
+
+  /**
+   * Clustering commit : cl
+   * Commit : c
+   * |requestTime<---commit type--->completionTime|(written partitions in current commit)
+   * ---------------------------------------------------------------------------------------------------------------------------------------------> timeline
+   * |0<-c1->1|(part=0000)
+   *           |2<------cl2------>4|(part=0000)
+   *                        |3<-----------c3--------->6|(part=0003)
+   *                                         |5<-----------c4--------->7|(part=0005)
+   *                                                                                |8<-------------------c5----------------->N|(part=0008)
+   *                                                                                                             | start incr clustering at 9
+   */
+  @Test
+  public void testExecutorWithMultiWriter() throws Exception {
+    String tableName = "testTable";
+    String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+    HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+        HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+        .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+        .withMarkersType("DIRECT")
+        .build();
+
+    // <requestTime, CompletionTime>
+    HashMap<String, String> instants = new HashMap<>();
+    instants.put("0000", "0001"); // commit 1
+    instants.put("0002", "0004"); // clustering 1
+    instants.put("0003", "0006"); // commit 2
+    instants.put("0005", "0007"); // commit 3
+
+    prepareTimeline(metaClient, instants, Collections.singletonList("0008"));
+
+    DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+    Set<String> incrementalPartitions = (Set<String>)executor.getIncrementalPartitions(TableServiceType.CLUSTER).getRight();
+
+    assertEquals(2, incrementalPartitions.size());
+    assertTrue(incrementalPartitions.contains("0003"));
+    assertTrue(incrementalPartitions.contains("0005"));
+    assertFalse(incrementalPartitions.contains("0008"));
+  }
+
+  @Test
+  public void testGetPartitionsFallbackToFullScan() throws Exception {
+    String tableName = "testTable";
+    String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+    HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+        HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+        .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+        .withMarkersType("DIRECT")
+        .build();
+
+    // <requestTime, CompletionTime>
+    HashMap<String, String> instants = new HashMap<>();
+    instants.put("0000", "0001");
+    instants.put("0002", "0004");
+    instants.put("0003", "0006");
+    instants.put("0005", "0007");
+    instants.put("0008", "0010");
+
+    prepareTimeline(metaClient, instants, Collections.emptyList());
+    instants.keySet().forEach(instant -> {
+      try {
+        FileCreateUtils.createPartitionMetaFile(tablePath, instant);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    });
+    DummyStrategy incrementalStrategy = new DummyStrategy();
+    DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+    List<String> partitions = (List<String>) executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+    assertEquals(2, partitions.size());
+
+    // Simulation archive commit instant 0000 and clustering instant 0002
+    Path path = new Path(tablePath);
+    FileSystem fs = path.getFileSystem(new Configuration());
+    StoragePath timelinePath = metaClient.getTimelinePath();
+    Arrays.stream(fs.listStatus(new Path(timelinePath.toString()))).forEach(instant -> {
+      if (instant.getPath().toString().contains("0000") || instant.getPath().toString().contains("0002")) {
+        try {
+          fs.delete(instant.getPath(), true);
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    });
+
+    // fall back to get all partitions
+    List<String> allPartitions = (List<String>) executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+    assertEquals(allPartitions.stream().sorted().collect(Collectors.toList()), instants.keySet().stream().sorted().collect(Collectors.toList()));
+  }
+
+  @Test
+  public void testContinuousEmptyCommits() throws Exception {
+    String tableName = "testTable";
+    String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+    HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+        HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+        .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+        .withMarkersType("DIRECT")
+        .build();
+    HoodieTestTable testTable = HoodieTestTable.of(metaClient);
+
+    // create clustering commit 0001_0002
+    String clusteringRequestTime = "0001";
+    String clusteringCompletionTIme = "0002";
+    Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> replaceMetadata =
+        generateReplaceCommitMetadata(clusteringRequestTime, clusteringRequestTime, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+    testTable.addCluster(clusteringRequestTime, replaceMetadata.getKey(), Option.empty(), replaceMetadata.getValue(), clusteringCompletionTIme);
+
+    HoodieInstant clusteringInstant = metaClient.getActiveTimeline().filterCompletedInstants().getLastClusteringInstant().get();
+    FileCreateUtils.createPartitionMetaFile(tablePath, clusteringInstant.requestedTime());
+
+    // create empty commit 0003_004
+    String emptyCommitRequestTime = "0003";
+    String emptyCommitCompletionTIme = "0004";
+    HoodieCommitMetadata metadata = testTable.createCommitMetadata(emptyCommitRequestTime, WriteOperationType.INSERT, Collections.emptyList(), 0, false);
+    testTable.addCommit(emptyCommitRequestTime, Option.of(emptyCommitCompletionTIme), Option.of(metadata));
+
+    // create empty commit 0005_0006
+    String emptyCommitRequestTime2 = "0005";
+    String emptyCommitCompletionTIme2 = "0006";
+    HoodieCommitMetadata metadata2 = testTable.createCommitMetadata(emptyCommitRequestTime2, WriteOperationType.INSERT, Collections.emptyList(), 0, false);
+    testTable.addCommit(emptyCommitRequestTime2, Option.of(emptyCommitCompletionTIme2), Option.of(metadata2));
+
+    // executor.getPartitions should return empty list
+    DummyStrategy incrementalStrategy = new DummyStrategy();
+    DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+    List partitions = executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+    assertTrue(partitions.isEmpty());
+  }
+
+  private void prepareTimeline(HoodieTableMetaClient metaClient, HashMap<String, String> commitInstants,
+                               List<String> requestInstants) throws Exception {
+    HoodieTestTable testTable = HoodieTestTable.of(metaClient);
+    commitInstants.forEach((requestTime, completionTime) -> {
+      try {
+        if (requestTime.equalsIgnoreCase("0002")) {
+          Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> replaceMetadata =
+              generateReplaceCommitMetadata(requestTime, requestTime, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+          testTable.addCluster(requestTime, replaceMetadata.getKey(), Option.empty(), replaceMetadata.getValue(), completionTime);
+        } else if (!requestTime.equalsIgnoreCase("0004")) {
+          HoodieCommitMetadata metadata = testTable.createCommitMetadata(requestTime, WriteOperationType.INSERT, Collections.singletonList(requestTime), 10, false);
+          testTable.addCommit(requestTime, Option.of(completionTime), Option.of(metadata));
+        }
+      } catch (Exception e) {
+        throw new HoodieException(e);
+      }
+    });
+
+    requestInstants.forEach(instant -> {
+      try {
+        testTable.addRequestedCommit(instant);
+        FileCreateUtils.createPartitionMetaFile(metaClient.getBasePath().toString(), instant);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+  }
+
+  private HoodieTable getMockHoodieTable(HoodieTableMetaClient metaClient) {
+    HoodieTable hoodieTable = mock(HoodieTable.class);
+    TaskContextSupplier taskContextSupplier = mock(TaskContextSupplier.class);
+    when(taskContextSupplier.getPartitionIdSupplier()).thenReturn(() -> 1);
+    when(hoodieTable.getTaskContextSupplier()).thenReturn(taskContextSupplier);
+    when(hoodieTable.getMetaClient()).thenReturn(metaClient);
+    when(hoodieTable.getActiveTimeline()).thenReturn(metaClient.getActiveTimeline());
+    return hoodieTable;
+  }
+
+  private Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> generateReplaceCommitMetadata(
+      String instantTime, String partition, String replacedFileId, String newFileId) {
+    HoodieRequestedReplaceMetadata requestedReplaceMetadata = new HoodieRequestedReplaceMetadata();
+    requestedReplaceMetadata.setOperationType(WriteOperationType.CLUSTER.toString());
+    requestedReplaceMetadata.setVersion(1);
+    HoodieSliceInfo sliceInfo = HoodieSliceInfo.newBuilder().setFileId(replacedFileId).build();
+    List<HoodieClusteringGroup> clusteringGroups = new ArrayList<>();
+    clusteringGroups.add(HoodieClusteringGroup.newBuilder()
+        .setVersion(1).setNumOutputFileGroups(1).setMetrics(Collections.emptyMap())
+        .setSlices(Collections.singletonList(sliceInfo)).build());
+    requestedReplaceMetadata.setExtraMetadata(Collections.emptyMap());
+    requestedReplaceMetadata.setClusteringPlan(HoodieClusteringPlan.newBuilder()
+        .setVersion(1).setExtraMetadata(Collections.emptyMap())
+        .setStrategy(HoodieClusteringStrategy.newBuilder().setStrategyClassName("").setVersion(1).build())
+        .setInputGroups(clusteringGroups).build());
+
+    HoodieReplaceCommitMetadata replaceMetadata = new HoodieReplaceCommitMetadata();
+    replaceMetadata.addReplaceFileId(partition, replacedFileId);
+    replaceMetadata.setOperationType(WriteOperationType.CLUSTER);
+    if (!StringUtils.isNullOrEmpty(newFileId)) {
+      HoodieWriteStat writeStat = new HoodieWriteStat();
+      writeStat.setPartitionPath(partition);
+      writeStat.setPath(partition + "/" + HoodieTestCommitGenerator.getBaseFilename(instantTime, newFileId));
+      writeStat.setFileId(newFileId);
+      writeStat.setTotalWriteBytes(1);
+      writeStat.setFileSizeInBytes(1);
+      replaceMetadata.addWriteStat(partition, writeStat);
+    }
+    return Pair.of(requestedReplaceMetadata, replaceMetadata);
+  }
+
+  class DummyTableServicePlanActionExecutor<T, I, K, O, R> extends BaseTableServicePlanActionExecutor<T, I, K, O, R> {
+
+    public DummyTableServicePlanActionExecutor(HoodieEngineContext context,
+                                               HoodieWriteConfig config,
+                                               HoodieTable<T, I, K, O> table,
+                                               String instantTime) {
+      super(context, config, table, instantTime);
+    }
+
+    @Override
+    public R execute() {
+      return null;
+    }
+  }
+
+  class DummyStrategy implements IncrementalPartitionAwareStrategy {
+
+    @Override
+    public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions) {
+      return null;
+    }
+  }
+}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
index a053a961105..e5ffda9fc53 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
@@ -18,7 +18,6 @@
 
 package org.apache.hudi.table.action.cluster.strategy;
 
-import org.apache.hudi.avro.model.HoodieClusteringGroup;
 import org.apache.hudi.common.engine.HoodieEngineContext;
 import org.apache.hudi.config.HoodieClusteringConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
@@ -32,7 +31,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -83,11 +81,6 @@ public class TestPartitionAwareClusteringPlanStrategy {
       super(table, engineContext, writeConfig);
     }
 
-    @Override
-    protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List list) {
-      return null;
-    }
-
     @Override
     protected Map<String, String> getStrategyParams() {
       return null;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
index a232bf54c16..a6684339b77 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
 import org.apache.hudi.common.model.HoodieKey;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -56,7 +57,7 @@ public class FlinkSizeBasedClusteringPlanStrategy<T>
   }
 
   @Override
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
   }
 
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
index 10046f84f5b..4edf816dd47 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
@@ -34,6 +34,8 @@ import org.apache.hudi.common.util.CollectionUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
+import org.apache.hudi.util.Lazy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,7 +63,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
   }
 
   @Override
-  public Option<HoodieClusteringPlan> generateClusteringPlan() {
+  public Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions) {
     if (!checkPrecondition()) {
       return Option.empty();
     }
@@ -71,7 +73,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
 
     List<String> partitionPaths = getPartitionPathInActiveTimeline(hoodieTable);
 
-    partitionPaths = filterPartitionPaths(partitionPaths);
+    partitionPaths = filterPartitionPaths(getWriteConfig(), partitionPaths).getLeft();
 
     if (partitionPaths.isEmpty()) {
       // In case no partitions could be picked, return no clustering plan
@@ -82,7 +84,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
             .flatMap(
                     partitionPaths, partitionPath -> {
                     List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
-                    return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).limit(getWriteConfig().getClusteringMaxNumGroups());
+                    return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).getLeft().limit(getWriteConfig().getClusteringMaxNumGroups());
                 },
                     partitionPaths.size())
             .stream()
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
index 36cb5ebdf90..e31570f7d02 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
@@ -74,7 +74,7 @@ public class TestFlinkSizeBasedClusteringPlanStrategy {
           .build())
         .build();
     PartitionAwareClusteringPlanStrategy strategyWithSortEnabled = new FlinkSizeBasedClusteringPlanStrategy(table, context, configWithSortEnabled);
-    Stream<HoodieClusteringGroup> groupStreamSort = strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+    Stream<HoodieClusteringGroup> groupStreamSort = (Stream<HoodieClusteringGroup>) strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
     assertEquals(1, groupStreamSort.count());
 
     // test buildClusteringGroupsForPartition without ClusteringSortColumns config
@@ -86,7 +86,7 @@ public class TestFlinkSizeBasedClusteringPlanStrategy {
           .build())
         .build();
     PartitionAwareClusteringPlanStrategy strategyWithSortDisabled = new FlinkSizeBasedClusteringPlanStrategy(table, context, configWithSortDisabled);
-    Stream<HoodieClusteringGroup> groupStreamWithOutSort = strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+    Stream<HoodieClusteringGroup> groupStreamWithOutSort = (Stream<HoodieClusteringGroup>) strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
     assertEquals(0, groupStreamWithOutSort.count());
   }
 
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
index 03d416452d2..e0309e50e75 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
 import org.apache.hudi.common.model.HoodieKey;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -57,7 +58,7 @@ public class JavaSizeBasedClusteringPlanStrategy<T>
   }
 
   @Override
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
   }
 
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
index ba3f917d04c..9e6a76e398a 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
@@ -43,13 +43,13 @@ public class SparkSingleFileSortPlanStrategy<T>
   }
 
   @Override
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     List<Pair<List<FileSlice>, Integer>> fileSliceGroups = fileSlices.stream()
         .map(fileSlice -> Pair.of(Collections.singletonList(fileSlice), 1)).collect(Collectors.toList());
-    return fileSliceGroups.stream().map(fileSliceGroup -> HoodieClusteringGroup.newBuilder()
+    return Pair.of(fileSliceGroups.stream().map(fileSliceGroup -> HoodieClusteringGroup.newBuilder()
         .setSlices(getFileSliceInfo(fileSliceGroup.getLeft()))
         .setNumOutputFileGroups(fileSliceGroup.getRight())
         .setMetrics(buildMetrics(fileSliceGroup.getLeft()))
-        .build());
+        .build()), true);
   }
 }
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
index 2c82810d175..0d3bac365bd 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
 import org.apache.hudi.common.model.HoodieKey;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -57,7 +58,7 @@ public class SparkSizeBasedClusteringPlanStrategy<T>
   }
 
   @Override
-  protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+  protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
     return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
   }
 
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
index ad9c5b45a0f..4d3d40b1799 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
@@ -67,7 +67,7 @@ public class TestSparkSizeBasedClusteringPlanStrategy {
     fileSlices.add(createFileSlice(400));
     fileSlices.add(createFileSlice(400));
 
-    Stream<HoodieClusteringGroup> clusteringGroupStream = planStrategy.buildClusteringGroupsForPartition("p0", fileSlices);
+    Stream<HoodieClusteringGroup> clusteringGroupStream = (Stream<HoodieClusteringGroup>) planStrategy.buildClusteringGroupsForPartition("p0", fileSlices).getLeft();
     List<HoodieClusteringGroup> clusteringGroups = clusteringGroupStream.collect(Collectors.toList());
 
     // FileSlices will be divided into two clusteringGroups
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
index aed4cff3ef1..a878cf43817 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
@@ -26,7 +26,10 @@ import org.apache.hudi.common.fs.ConsistencyGuardConfig;
 import org.apache.hudi.common.model.FileSlice;
 import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2;
 import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
 import org.apache.hudi.common.testutils.CompactionTestUtils;
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
@@ -111,6 +114,16 @@ public class ClusteringTestUtils {
     return runClusteringOnInstant(clusteringClient, skipExecution, shouldCommit, clusteringCommitTime);
   }
 
+  public static HoodieClusteringPlan getClusteringPlan(HoodieTableMetaClient client, HoodieInstant clusteringInstant) {
+    return ClusteringUtils.getClusteringPlan(client, clusteringInstant).get().getRight();
+  }
+
+  public static HoodieClusteringPlan getClusteringPlan(HoodieTableMetaClient client, String clusteringInstantString) {
+    HoodieInstant clusteringInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.CLUSTERING_ACTION,
+        clusteringInstantString, InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+    return getClusteringPlan(client, clusteringInstant);
+  }
+
   public static String runClusteringOnInstant(SparkRDDWriteClient clusteringClient, boolean skipExecution, boolean shouldCommit, String clusteringCommitTime) {
     clusteringClient.scheduleClusteringAtInstant(clusteringCommitTime, Option.empty());
     if (!skipExecution) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java
new file mode 100644
index 00000000000..970954e0637
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieSliceInfo;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.config.HoodieClusteringConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.table.HoodieSparkTable;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.strategy.ClusteringPlanStrategy;
+import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.config.HoodieClusteringConfig.DAYBASED_LOOKBACK_PARTITIONS;
+import static org.apache.hudi.config.HoodieClusteringConfig.PARTITION_FILTER_BEGIN_PARTITION;
+import static org.apache.hudi.config.HoodieClusteringConfig.PARTITION_FILTER_END_PARTITION;
+import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_PARTITION_FILTER_MODE;
+import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestIncrementalClustering extends SparkClientFunctionalTestHarness {
+
+  private static final String TODAY;
+  private static final String YESTERDAY;
+  private static final String TOMORROW;
+
+  static {
+    LocalDate today = LocalDate.now();
+    LocalDate yesterday = today.minusDays(1);
+    LocalDate tomorrow = today.plusDays(1);
+    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
+
+    TODAY = today.format(formatter);
+    YESTERDAY = yesterday.format(formatter);
+    TOMORROW = tomorrow.format(formatter);
+  }
+
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testPartitionsForIncrClusteringWithMaxGroupLimit(boolean enableIncrTableService) throws Exception {
+    int maxClusteringGroup = 1;
+    HoodieWriteConfig writeConfig = buildWriteConfig(enableIncrTableService, new Properties(), maxClusteringGroup);
+    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.COPY_ON_WRITE);
+    String[] partitions = {YESTERDAY, TODAY};
+    // write twice, make sure there are multi-slices in each partitions
+    prepareBasicData(writeConfig, partitions);
+    prepareBasicData(writeConfig, partitions);
+    String clusteringInstantTime = doClustering(writeConfig);
+
+    HoodieClusteringPlan clusteringPlan = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime);
+    if (enableIncrTableService) {
+      // Since setting maxClusteringGroup 1, and there are 2 slices under each partition,
+      // all files in each partition have not been completely processed, that is, they are all missing partitions.
+      assertEquals(2, clusteringPlan.getMissingSchedulePartitions().size());
+      assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(YESTERDAY));
+      assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(TODAY));
+
+      String[] partitions2 = {TOMORROW};
+      prepareBasicData(buildWriteConfig(true, new Properties(), maxClusteringGroup), partitions2);
+      HoodieSparkTable table = HoodieSparkTable.create(writeConfig, context());
+      ClusteringPlanActionExecutor executor =
+          new ClusteringPlanActionExecutor<>(context(), writeConfig, table, "9999999", Option.empty());
+
+      List<String> incrementalPartitions = executor.getPartitions(ReflectionUtils.loadClass(
+          ClusteringPlanStrategy.checkAndGetClusteringPlanStrategy(writeConfig),
+          new Class<?>[] {HoodieTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, table, context(), writeConfig), TableServiceType.CLUSTER);
+      // fetch 2 missing partitions from last completed clustering plan as incremental partitions
+      // get 1 partitions from new commit as incremental partitions
+      assertEquals(3, incrementalPartitions.size());
+    } else {
+      assertNull(clusteringPlan.getMissingSchedulePartitions());
+    }
+  }
+
+  @ParameterizedTest
+  @MethodSource("testIncrClusteringWithFilter")
+  public void testPartitionsForIncrClusteringWithFilter(ClusteringPlanPartitionFilterMode mode,
+                                                        Properties props) throws Exception {
+    HoodieWriteConfig writeConfig = buildWriteConfig(true, props, 100);
+    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.COPY_ON_WRITE);
+    String[] partitions = {YESTERDAY, TODAY};
+    // write twice, make sure there are multi-slices in each partitions
+    prepareBasicData(writeConfig, partitions);
+    String clusteringInstantTime = doClustering(writeConfig);
+    HoodieClusteringPlan clusteringPlan = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime);
+
+    switch (mode) {
+      case NONE: {
+        // For partitions filtered out by the regex expression, they will not be recorded in the missingPartitions
+        assertEquals(0, clusteringPlan.getMissingSchedulePartitions().size());
+        break;
+      }
+      case SELECTED_PARTITIONS: {
+        Set<String> affectedPartitions = getAffectedPartition(clusteringPlan);
+        assertEquals(1, affectedPartitions.size());
+        assertTrue(affectedPartitions.contains(YESTERDAY));
+        assertEquals(0, clusteringPlan.getMissingSchedulePartitions().size());
+        break;
+      }
+      case RECENT_DAYS: {
+        assertEquals(1, clusteringPlan.getMissingSchedulePartitions().size());
+        assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(TODAY));
+        // do another TOMORROW ingestion and clustering
+        String[] partitions2 = {TOMORROW};
+        HoodieWriteConfig hoodieWriteConfig2 = buildWriteConfig(true, props, 100);
+        prepareBasicData(hoodieWriteConfig2, partitions2);
+        String clusteringInstantTime2 = doClustering(hoodieWriteConfig2);
+        HoodieClusteringPlan clusteringPlan2 = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime2);
+        assertEquals(1, clusteringPlan2.getMissingSchedulePartitions().size());
+        assertTrue(clusteringPlan2.getMissingSchedulePartitions().contains(TOMORROW));
+        break;
+      }
+      default: {
+        throw new HoodieException("Un-support mode" + mode);
+      }
+    }
+  }
+
+  private Set<String> getAffectedPartition(HoodieClusteringPlan clusteringPlan) {
+    return clusteringPlan.getInputGroups().stream().flatMap(hoodieClusteringGroup -> {
+      return hoodieClusteringGroup.getSlices().stream();
+    }).map(HoodieSliceInfo::getPartitionPath).collect(Collectors.toSet());
+  }
+
+  public static Stream<Object> testIncrClusteringWithFilter() {
+    Properties none = new Properties();
+    none.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.NONE);
+    none.put("hoodie.clustering.plan.strategy.partition.regex.pattern", TODAY + ".*");
+
+    Properties selectedPartitions = new Properties();
+    selectedPartitions.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.SELECTED_PARTITIONS.name());
+    selectedPartitions.put(PARTITION_FILTER_BEGIN_PARTITION.key(), YESTERDAY);
+    selectedPartitions.put(PARTITION_FILTER_END_PARTITION.key(), YESTERDAY);
+
+    Properties recentDay = new Properties();
+    recentDay.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.RECENT_DAYS);
+    recentDay.put(PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST.key(), 1);
+    recentDay.put(DAYBASED_LOOKBACK_PARTITIONS.key(), 1);
+
+    return Stream.of(
+        Arguments.of(ClusteringPlanPartitionFilterMode.NONE, none),
+        Arguments.of(ClusteringPlanPartitionFilterMode.SELECTED_PARTITIONS, selectedPartitions),
+        Arguments.of(ClusteringPlanPartitionFilterMode.RECENT_DAYS, recentDay)
+    );
+  }
+
+  private HoodieWriteConfig buildWriteConfig(boolean enableIncrTableService, Properties properties, int maxClusteringGroup) {
+    properties.put("hoodie.datasource.write.row.writer.enable", String.valueOf(false));
+    properties.put("hoodie.parquet.small.file.limit", String.valueOf(-1));
+    return getConfigBuilder(true)
+        .withIncrementalTableServiceEnabled(enableIncrTableService)
+        .withClusteringConfig(HoodieClusteringConfig.newBuilder()
+            .withClusteringMaxNumGroups(maxClusteringGroup)
+            .withClusteringMaxBytesInGroup(maxClusteringGroup)
+            .fromProperties(properties)
+            .build())
+        .withProperties(properties)
+        .build();
+  }
+
+  private void prepareBasicData(HoodieWriteConfig writeConfig, String[] partitions) throws IOException {
+    SparkRDDWriteClient client = getHoodieWriteClient(writeConfig);
+    HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(partitions);
+    for (int i = 0; i < partitions.length; i++) {
+      String instantTime = client.createNewInstantTime();
+      client.startCommitWithTime(instantTime);
+      client.insert(jsc().parallelize(dataGen.generateInsertsForPartition(instantTime, 10, partitions[i]), 1), instantTime);
+    }
+    client.close();
+  }
+
+  private String doClustering(HoodieWriteConfig writeConfig) throws IOException {
+    try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
+      return ClusteringTestUtils.runClustering(client, false, true);
+    }
+  }
+}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
index ada5f4954ab..5e31d2ee4f3 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
@@ -73,7 +73,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
           .build())
         .build();
     PartitionAwareClusteringPlanStrategy strategyWithSortEnabled = new SparkSizeBasedClusteringPlanStrategy(table, context, configWithSortEnabled);
-    Stream<HoodieClusteringGroup> groupStreamSort = strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+    Stream<HoodieClusteringGroup> groupStreamSort = (Stream<HoodieClusteringGroup>) strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
     assertEquals(1, groupStreamSort.count());
 
     // test buildClusteringGroupsForPartition without ClusteringSortColumns config
@@ -85,7 +85,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
           .build())
         .build();
     PartitionAwareClusteringPlanStrategy strategyWithSortDisabled = new SparkSizeBasedClusteringPlanStrategy(table, context, configWithSortDisabled);
-    Stream<HoodieClusteringGroup> groupStreamWithOutSort = strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+    Stream<HoodieClusteringGroup> groupStreamWithOutSort = (Stream<HoodieClusteringGroup>) strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
     assertEquals(0, groupStreamWithOutSort.count());
   }
 
@@ -107,7 +107,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
                 .build())
         .build();
     PartitionAwareClusteringPlanStrategy clusteringPlanStrategy = new SparkSizeBasedClusteringPlanStrategy(table, context, writeConfig);
-    Stream<HoodieClusteringGroup> groups = clusteringPlanStrategy.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+    Stream<HoodieClusteringGroup> groups = (Stream<HoodieClusteringGroup>) clusteringPlanStrategy.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
     assertEquals(2, groups.count());
   }
 
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
index 70643a327d4..a173f4df80b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
@@ -63,7 +63,7 @@ public class TestSparkClusteringPlanPartitionFilter {
     fakeTimeBasedPartitionsPath.add("20210718");
     fakeTimeBasedPartitionsPath.add("20210716");
     fakeTimeBasedPartitionsPath.add("20210719");
-    List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+    List list = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
     assertEquals(3, list.size());
   }
 
@@ -81,7 +81,7 @@ public class TestSparkClusteringPlanPartitionFilter {
     fakeTimeBasedPartitionsPath.add("20210718");
     fakeTimeBasedPartitionsPath.add("20210716");
     fakeTimeBasedPartitionsPath.add("20210719");
-    List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+    List list = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
     assertEquals(1, list.size());
     assertSame("20210718", list.get(0));
   }
@@ -101,7 +101,7 @@ public class TestSparkClusteringPlanPartitionFilter {
     fakeTimeBasedPartitionsPath.add("20211221");
     fakeTimeBasedPartitionsPath.add("20211222");
     fakeTimeBasedPartitionsPath.add("20211224");
-    List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+    List list = (List)sg.filterPartitionPaths(table.getConfig(), fakeTimeBasedPartitionsPath).getLeft();
     assertEquals(1, list.size());
     assertSame("20211222", list.get(0));
   }
@@ -117,7 +117,7 @@ public class TestSparkClusteringPlanPartitionFilter {
     for (int i = 0; i < 24; i++) {
       fakeTimeBasedPartitionsPath.add("20220301" + (i >= 10 ? String.valueOf(i) : "0" + i));
     }
-    List filterPartitions = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+    List filterPartitions = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
     assertEquals(1, filterPartitions.size());
     assertEquals(fakeTimeBasedPartitionsPath.get(DateTime.now().getHourOfDay()), filterPartitions.get(0));
     fakeTimeBasedPartitionsPath = new ArrayList<>();
@@ -125,7 +125,7 @@ public class TestSparkClusteringPlanPartitionFilter {
       fakeTimeBasedPartitionsPath.add("20220301" + (i >= 10 ? String.valueOf(i) : "0" + i));
       fakeTimeBasedPartitionsPath.add("20220302" + (i >= 10 ? String.valueOf(i) : "0" + i));
     }
-    filterPartitions = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+    filterPartitions = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
     assertEquals(2, filterPartitions.size());
 
     int hourOfDay = DateTime.now().getHourOfDay();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
index a368d43da25..8187b22bd9a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
@@ -30,6 +30,8 @@ import org.apache.hudi.config.HoodieCompactionConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -52,8 +54,9 @@ public class TestHoodieCompactionStrategy {
   private static final Random RANDOM = new Random();
   private String[] partitionPaths = {"2017/01/01", "2017/01/02", "2017/01/03"};
 
-  @Test
-  public void testUnBounded() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testUnBounded(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -61,26 +64,39 @@ public class TestHoodieCompactionStrategy {
     sizesMap.put(90 * MB, Collections.singletonList(1024 * MB));
     UnBoundedCompactionStrategy strategy = new UnBoundedCompactionStrategy();
     HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp")
+        .withIncrementalTableServiceEnabled(enableIncrTableService)
         .withCompactionConfig(HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).build()).build();
-    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap).getLeft();
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy()
+        .orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+
     assertEquals(operations, returned, "UnBounded should not re-order or filter");
+    assertEquals(missingPartitions.size(), 0);
   }
 
-  @Test
-  public void testBoundedIOSimple() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testBoundedIOSimple(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
     sizesMap.put(100 * MB, Collections.singletonList(MB));
     sizesMap.put(90 * MB, Collections.singletonList(1024 * MB));
     BoundedIOCompactionStrategy strategy = new BoundedIOCompactionStrategy();
-    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
+    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp")
+        .withIncrementalTableServiceEnabled(enableIncrTableService)
+        .withCompactionConfig(
             HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(400).build())
         .build();
-    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
-
+    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap).getLeft();
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      assertEquals(1, missingPartitions.stream().distinct().count());
+    }
     assertTrue(returned.size() < operations.size(), "BoundedIOCompaction should have resulted in fewer compactions");
     assertEquals(2, returned.size(), "BoundedIOCompaction should have resulted in 2 compactions being chosen");
     // Total size of all the log files
@@ -90,8 +106,9 @@ public class TestHoodieCompactionStrategy {
         "Should chose the first 2 compactions which should result in a total IO of 610 MB");
   }
 
-  @Test
-  public void testLogFileSizeCompactionSimple() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testLogFileSizeCompactionSimple(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -102,9 +119,17 @@ public class TestHoodieCompactionStrategy {
             HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(1205)
                 .withLogFileSizeThresholdBasedCompaction(100 * 1024 * 1024).build())
         .build();
-    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
-
+    Pair<List<HoodieCompactionOperation>, Map<Long, String>> operationAndPartition = createCompactionOperations(writeConfig, sizesMap);
+    List<HoodieCompactionOperation> operations = operationAndPartition.getLeft();
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      String missedPart1 = operationAndPartition.getRight().get(110 * MB);
+      String missedPart2 = operationAndPartition.getRight().get(100 * MB);
+      assertTrue(missingPartitions.contains(missedPart1));
+      assertTrue(missingPartitions.contains(missedPart2));
+    }
     assertTrue(returned.size() < operations.size(),
         "LogFileSizeBasedCompactionStrategy should have resulted in fewer compactions");
     assertEquals(2, returned.size(), "LogFileSizeBasedCompactionStrategy should have resulted in 2 compaction");
@@ -115,8 +140,9 @@ public class TestHoodieCompactionStrategy {
         "Should chose the first 2 compactions which should result in a total IO of 1594 MB");
   }
 
-  @Test
-  public void testDayBasedCompactionSimple() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testDayBasedCompactionSimple(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -134,10 +160,16 @@ public class TestHoodieCompactionStrategy {
 
     DayBasedCompactionStrategy strategy = new DayBasedCompactionStrategy();
     HoodieWriteConfig writeConfig =
-        HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+        HoodieWriteConfig.newBuilder().withPath("/tmp")
+            .withIncrementalTableServiceEnabled(enableIncrTableService)
+            .withCompactionConfig(HoodieCompactionConfig.newBuilder()
             .withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(1).build()).build();
-
-    List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+    Pair<List<String>, List<String>> resPair = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+    List<String> filterPartitions = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      assertTrue(missingPartitions.isEmpty());
+    }
     assertEquals(1, filterPartitions.size(), "DayBasedCompactionStrategy should have resulted in fewer partitions");
 
     List<HoodieCompactionOperation> operations = createCompactionOperationsForPartition(writeConfig, sizesMap, keyToPartitionMap, filterPartitions);
@@ -155,8 +187,9 @@ public class TestHoodieCompactionStrategy {
     assertTrue(comparison >= 0, "DayBasedCompactionStrategy should sort partitions in descending order");
   }
 
-  @Test
-  public void testDayBasedCompactionWithIOBounded() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testDayBasedCompactionWithIOBounded(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -175,6 +208,7 @@ public class TestHoodieCompactionStrategy {
     DayBasedCompactionStrategy strategy = new DayBasedCompactionStrategy();
     HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
         .withPath("/tmp")
+        .withIncrementalTableServiceEnabled(enableIncrTableService)
         .withCompactionConfig(HoodieCompactionConfig.newBuilder()
             .withCompactionStrategy(strategy)
             .withTargetPartitionsPerDayBasedCompaction(1)
@@ -182,11 +216,17 @@ public class TestHoodieCompactionStrategy {
             .build())
         .build();
 
-    List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+    List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths)).getLeft();
     assertEquals(1, filterPartitions.size(), "DayBasedCompactionStrategy should have resulted in fewer partitions");
 
     List<HoodieCompactionOperation> operations = createCompactionOperationsForPartition(writeConfig, sizesMap, keyToPartitionMap, filterPartitions);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      assertEquals(1, missingPartitions.size());
+      assertTrue(missingPartitions.contains(partitionPaths[2]));
+    }
 
     assertEquals(1, returned.size(),
         "DayBasedAndBoundedIOCompactionStrategy should have resulted in fewer compactions");
@@ -204,9 +244,10 @@ public class TestHoodieCompactionStrategy {
     assertEquals(390, (long) returnedSize,
         "Should chose the first and the third compactions which should result in a total IO of 591 MB");
   }
-  
-  @Test
-  public void testBoundedPartitionAwareCompactionSimple() {
+
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testBoundedPartitionAwareCompactionSimple(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -238,10 +279,17 @@ public class TestHoodieCompactionStrategy {
 
     BoundedPartitionAwareCompactionStrategy strategy = new BoundedPartitionAwareCompactionStrategy();
     HoodieWriteConfig writeConfig =
-        HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+        HoodieWriteConfig.newBuilder().withPath("/tmp")
+            .withIncrementalTableServiceEnabled(enableIncrTableService)
+            .withCompactionConfig(HoodieCompactionConfig.newBuilder()
             .withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(2).build()).build();
     List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      assertTrue(missingPartitions.isEmpty());
+    }
 
     assertTrue(returned.size() < operations.size(),
         "BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions");
@@ -254,8 +302,9 @@ public class TestHoodieCompactionStrategy {
     assertTrue(comparison >= 0, "BoundedPartitionAwareCompactionStrategy should sort partitions in descending order");
   }
 
-  @Test
-  public void testUnboundedPartitionAwareCompactionSimple() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testUnboundedPartitionAwareCompactionSimple(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -287,10 +336,17 @@ public class TestHoodieCompactionStrategy {
 
     UnBoundedPartitionAwareCompactionStrategy strategy = new UnBoundedPartitionAwareCompactionStrategy();
     HoodieWriteConfig writeConfig =
-        HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+        HoodieWriteConfig.newBuilder().withPath("/tmp")
+            .withIncrementalTableServiceEnabled(enableIncrTableService)
+            .withCompactionConfig(HoodieCompactionConfig.newBuilder()
             .withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(2).build()).build();
     List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      assertTrue(missingPartitions.isEmpty());
+    }
 
     assertTrue(returned.size() < operations.size(),
         "UnBoundedPartitionAwareCompactionStrategy should not include last "
@@ -299,8 +355,9 @@ public class TestHoodieCompactionStrategy {
         "BoundedPartitionAwareCompactionStrategy should have resulted in 1 compaction");
   }
 
-  @Test
-  public void testLogFileLengthBasedCompactionStrategy() {
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testLogFileLengthBasedCompactionStrategy(boolean enableIncrTableService) {
     Map<Long, List<Long>> sizesMap = new HashMap<>();
     sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, new ArrayList<>());
@@ -311,8 +368,16 @@ public class TestHoodieCompactionStrategy {
             HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(1024)
                 .withCompactionLogFileNumThreshold(2).build())
         .build();
-    List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
-    List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    Pair<List<HoodieCompactionOperation>, Map<Long, String>> operationAndPartition = createCompactionOperations(writeConfig, sizesMap);
+    List<HoodieCompactionOperation> operations = operationAndPartition.getLeft();
+    Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+    List<HoodieCompactionOperation> returned = resPair.getLeft();
+    List<String> missingPartitions = resPair.getRight();
+    if (enableIncrTableService) {
+      Map<Long, String> partition = operationAndPartition.getRight();
+      assertTrue(missingPartitions.contains(partition.get(110 * MB)));
+      assertTrue(missingPartitions.contains(partition.get(100 * MB)));
+    }
 
     assertTrue(returned.size() < operations.size(),
         "LogFileLengthBasedCompactionStrategy should have resulted in fewer compactions");
@@ -341,7 +406,7 @@ public class TestHoodieCompactionStrategy {
     List<String> allPartitionPaths = Arrays.asList(
         "2017/01/01", "2018/01/02", "2017/02/01"
     );
-    List<String> returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths);
+    List<String> returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths).getLeft();
     // filter by num first and then filter by prefix
     assertEquals(1, returned.size());
     assertEquals("2017/01/01", returned.get(0));
@@ -349,7 +414,7 @@ public class TestHoodieCompactionStrategy {
     writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
         HoodieCompactionConfig.newBuilder().withCompactionStrategy(new PrefixStrategy(), new NumStrategy()).withTargetIOPerCompactionInMB(1024)
             .withCompactionLogFileNumThreshold(2).build()).build();
-    returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths);
+    returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths).getLeft();
     // filter by prefix first and then filter by num
     assertEquals(2, returned.size());
     assertEquals("2017/01/01", returned.get(0));
@@ -358,15 +423,28 @@ public class TestHoodieCompactionStrategy {
 
   public static class NumStrategy extends CompactionStrategy {
     @Override
-    public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
-      return allPartitionPaths.stream().limit(2).collect(Collectors.toList());
+    public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+      List<String> partitionToProcess = allPartitionPaths.stream().limit(2).collect(Collectors.toList());
+      List<String> missingPartitions = allPartitionPaths.stream().skip(2).collect(Collectors.toList());
+      return Pair.of(partitionToProcess, missingPartitions);
     }
   }
 
   public static class PrefixStrategy extends CompactionStrategy {
     @Override
-    public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
-      return allPartitionPaths.stream().filter(s -> s.startsWith("2017")).collect(Collectors.toList());
+    public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+      ArrayList<String> partitionsToProcess = new ArrayList<>();
+      ArrayList<String> missingPartitions = new ArrayList<>();
+
+      allPartitionPaths.forEach(partition -> {
+        if (partition.startsWith("2017")) {
+          partitionsToProcess.add(partition);
+        } else {
+          missingPartitions.add(partition);
+        }
+      });
+
+      return Pair.of(partitionsToProcess, missingPartitions);
     }
   }
 
@@ -381,21 +459,21 @@ public class TestHoodieCompactionStrategy {
     );
 
     HoodieWriteConfig writeConfig = updateRegex(".*");
-    List<String> filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    List<String> filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(5, filteredPartitions.size());
 
     writeConfig = updateRegex("2020/01/01");
-    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(1, filteredPartitions.size());
     assertEquals("2020/01/01", filteredPartitions.get(0));
 
     writeConfig = updateRegex("2020/01/0[1-2]");
-    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(2, filteredPartitions.size());
     assertEquals("2020/01/01", filteredPartitions.get(0));
     assertEquals("2020/01/02", filteredPartitions.get(1));
     writeConfig = updateRegex("2020/01/0[1-2]|2020/02/01");
-    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(3, filteredPartitions.size());
     assertEquals("2020/01/01", filteredPartitions.get(0));
     assertEquals("2020/01/02", filteredPartitions.get(1));
@@ -403,13 +481,13 @@ public class TestHoodieCompactionStrategy {
 
 
     writeConfig = updateRegex("2020/.*/01");
-    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(2, filteredPartitions.size());
     assertEquals("2020/01/01", filteredPartitions.get(0));
     assertEquals("2020/02/01", filteredPartitions.get(1));
 
     writeConfig = updateRegex(".*/01/.*");
-    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+    filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
     assertEquals(4, filteredPartitions.size());
     assertEquals("2020/01/01", filteredPartitions.get(0));
     assertEquals("2020/01/02", filteredPartitions.get(1));
@@ -426,12 +504,12 @@ public class TestHoodieCompactionStrategy {
     return writeConfig;
   }
 
-  private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,
+  private Pair<List<HoodieCompactionOperation>, Map<Long, String>> createCompactionOperations(HoodieWriteConfig config,
                                                                      Map<Long, List<Long>> sizesMap) {
     Map<Long, String> keyToPartitionMap = sizesMap.keySet().stream()
         .map(e -> Pair.of(e, partitionPaths[RANDOM.nextInt(partitionPaths.length - 1)]))
         .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
-    return createCompactionOperations(config, sizesMap, keyToPartitionMap);
+    return Pair.of(createCompactionOperations(config, sizesMap, keyToPartitionMap), keyToPartitionMap);
   }
 
   private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,
diff --git a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
index 87486267d1c..d4a6c207762 100644
--- a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
+++ b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
@@ -50,6 +50,14 @@
       "name":"preserveHoodieMetadata",
       "type":["null", "boolean"],
       "default": null
-    }
+    },
+     {
+       "name":"missingSchedulePartitions",
+       "type":["null", {
+         "type":"array",
+         "items":"string"
+       }],
+       "default": null
+     }
   ]
 }
diff --git a/hudi-common/src/main/avro/HoodieCompactionOperation.avsc b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
index bab7321f29c..61a96faec87 100644
--- a/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
+++ b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
@@ -96,6 +96,14 @@
        "name":"preserveHoodieMetadata",
        "type":["boolean", "null"],
        "default": false
+    },
+    {
+       "name":"missingSchedulePartitions",
+       "type":["null", {
+          "type":"array",
+          "items":"string"
+       }],
+       "default": null
     }
   ]
 }
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
index 2bc8f7fba62..eaed3668c9f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
@@ -192,7 +192,7 @@ public class CompactionTestUtils {
       }
     }).collect(Collectors.toList());
     return new HoodieCompactionPlan(ops.isEmpty() ? null : ops, new HashMap<>(),
-        CompactionUtils.LATEST_COMPACTION_METADATA_VERSION, null, null);
+        CompactionUtils.LATEST_COMPACTION_METADATA_VERSION, null, null, null);
   }
 
   /**
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
index f2bafc575ee..ac1a1a3dd10 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -305,6 +305,11 @@ public class FileCreateUtils {
         serializeCommitMetadata(commitMetadataSerDe, metadata).get());
   }
 
+  public static void createReplaceCommit(CommitMetadataSerDe commitMetadataSerDe, String basePath,
+                                         String instantTime, String completionTime, HoodieReplaceCommitMetadata metadata) throws IOException {
+    createMetaFile(basePath, instantTime, () -> completionTime, HoodieTimeline.REPLACE_COMMIT_EXTENSION, serializeCommitMetadata(commitMetadataSerDe, metadata).get());
+  }
+
   public static void createRequestedClusterCommit(String basePath, String instantTime,
                                                   HoodieRequestedReplaceMetadata requestedReplaceMetadata)
       throws IOException {
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
index f35a2792731..c40dd2350e5 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
@@ -131,7 +131,7 @@ public class TestClusteringUtil {
   private String generateClusteringPlan() {
     HoodieClusteringGroup group = new HoodieClusteringGroup();
     HoodieClusteringPlan plan = new HoodieClusteringPlan(Collections.singletonList(group),
-        HoodieClusteringStrategy.newBuilder().build(), Collections.emptyMap(), 1, false);
+        HoodieClusteringStrategy.newBuilder().build(), Collections.emptyMap(), 1, false, null);
     HoodieRequestedReplaceMetadata metadata = new HoodieRequestedReplaceMetadata(WriteOperationType.CLUSTER.name(),
         plan, Collections.emptyMap(), 1);
     String instantTime = table.getMetaClient().createNewInstantTime();
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
index 111680bdcdb..ccfd5965cad 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
@@ -174,7 +174,7 @@ public class TestCompactionUtil {
    */
   private String generateCompactionPlan() {
     HoodieCompactionOperation operation = new HoodieCompactionOperation();
-    HoodieCompactionPlan plan = new HoodieCompactionPlan(Collections.singletonList(operation), Collections.emptyMap(), 1, null, null);
+    HoodieCompactionPlan plan = new HoodieCompactionPlan(Collections.singletonList(operation), Collections.emptyMap(), 1, null, null, null);
     String instantTime = table.getMetaClient().createNewInstantTime();
     HoodieInstant compactionInstant =
         INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instantTime);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index c3e221b28e7..5353898136f 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -376,6 +376,19 @@ public class HoodieTestTable implements AutoCloseable {
     return this;
   }
 
+  public HoodieTestTable addCluster(
+      String instantTime,
+      HoodieRequestedReplaceMetadata requestedReplaceMetadata,
+      Option<HoodieCommitMetadata> inflightReplaceMetadata,
+      HoodieReplaceCommitMetadata completeReplaceMetadata,
+      String completionTime) throws Exception {
+    createRequestedClusterCommit(basePath, instantTime, requestedReplaceMetadata);
+    createInflightClusterCommit(COMMIT_METADATA_SER_DE, basePath, instantTime, inflightReplaceMetadata);
+    createReplaceCommit(COMMIT_METADATA_SER_DE, basePath, instantTime, completionTime, completeReplaceMetadata);
+    currentInstantTime = instantTime;
+    return this;
+  }
+
   public HoodieTestTable addRequestedReplace(String instantTime, Option<HoodieRequestedReplaceMetadata> requestedReplaceMetadata) throws Exception {
     createRequestedReplaceCommit(basePath, instantTime, requestedReplaceMetadata);
     currentInstantTime = instantTime;
@@ -1324,6 +1337,10 @@ public class HoodieTestTable implements AutoCloseable {
                                                                              HoodieTableType tableType,
                                                                              String commitTime,
                                                                              Map<String, List<Pair<String, Integer>>> partitionToFilesNameLengthMap) {
+    if (partitionToFilesNameLengthMap.isEmpty()) {
+      return testTableState;
+    }
+
     for (Map.Entry<String, List<Pair<String, Integer>>> partitionEntry : partitionToFilesNameLengthMap.entrySet()) {
       String partitionName = partitionEntry.getKey();
       List<Pair<String, Integer>> fileNameAndLengthList = partitionEntry.getValue();
@@ -1343,6 +1360,10 @@ public class HoodieTestTable implements AutoCloseable {
   public static List<HoodieWriteStat> generateHoodieWriteStatForPartition(Map<String, List<Pair<String, Integer>>> partitionToFileIdMap,
                                                                           String commitTime, boolean bootstrap) {
     List<HoodieWriteStat> writeStats = new ArrayList<>();
+    if (partitionToFileIdMap == null || partitionToFileIdMap.isEmpty()) {
+      return writeStats;
+    }
+
     for (Map.Entry<String, List<Pair<String, Integer>>> entry : partitionToFileIdMap.entrySet()) {
       String partition = entry.getKey();
       for (Pair<String, Integer> fileIdInfo : entry.getValue()) {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
index e01bbf4f9e3..1d085c006d7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
@@ -19,6 +19,7 @@
 
 package org.apache.hudi.table.action.compact;
 
+import org.apache.hudi.avro.model.HoodieCompactionOperation;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
 import org.apache.hudi.client.SparkRDDWriteClient;
 import org.apache.hudi.common.config.HoodieMemoryConfig;
@@ -28,10 +29,13 @@ import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.HoodieWriteStat;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieInstant.State;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
 import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.CompactionUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.config.HoodieCompactionConfig;
 import org.apache.hudi.config.HoodieIndexConfig;
@@ -47,6 +51,7 @@ import org.apache.hudi.table.HoodieSparkTable;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.HoodieWriteMetadata;
 import org.apache.hudi.table.action.compact.strategy.PartitionRegexBasedCompactionStrategy;
+import org.apache.hudi.table.action.compact.strategy.SmallBoundedIOCompactionStrategy;
 import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
 
 import com.codahale.metrics.Counter;
@@ -261,7 +266,7 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
         .build();
 
     try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
-      String newCommitTime = "100";
+      String newCommitTime = writeClient.createNewInstantTime();
       writeClient.startCommitWithTime(newCommitTime);
 
       List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
@@ -271,12 +276,12 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
       // trigger 2 updates following with compaction
       for (int i = 1; i < 5; i += 2) {
         // Update all the 100 records
-        newCommitTime = "10" + i;
+        newCommitTime = writeClient.createNewInstantTime();
         updateRecords(config, newCommitTime, records);
 
         assertLogFilesNumEqualsTo(config, 1);
 
-        HoodieWriteMetadata result = compact(writeClient, "10" + (i + 1));
+        HoodieWriteMetadata result = compact(writeClient, writeClient.createNewInstantTime());
         verifyCompaction(result);
 
         // Verify compaction.requested, compaction.completed metrics counts.
@@ -314,7 +319,7 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
         .withCompactionConfig(builder.build())
         .withMetricsConfig(getMetricsConfig()).build();
     try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
-      String newCommitTime = "100";
+      String newCommitTime = writeClient.createNewInstantTime();
       writeClient.startCommitWithTime(newCommitTime);
 
       List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 10);
@@ -322,18 +327,19 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
       writeClient.insert(recordsRDD, newCommitTime).collect();
 
       // update 1 time
-      newCommitTime = "101";
+      newCommitTime = writeClient.createNewInstantTime();
       updateRecords(config, newCommitTime, records);
       assertLogFilesNumEqualsTo(config, 1);
 
       // schedule compaction
-      boolean scheduled = writeClient.scheduleCompactionAtInstant("102", Option.empty());
+      String compactionInstant = writeClient.createNewInstantTime();
+      boolean scheduled = writeClient.scheduleCompactionAtInstant(compactionInstant, Option.empty());
       if (expectedCompactedPartition.isEmpty()) {
         assertFalse(scheduled);
         return;
       }
 
-      HoodieWriteMetadata result = compact(writeClient, "102");
+      HoodieWriteMetadata result = compact(writeClient, compactionInstant);
 
       assertTrue(result.getWriteStats().isPresent());
       List<HoodieWriteStat> stats = (List<HoodieWriteStat>) result.getWriteStats().get();
@@ -344,6 +350,82 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
     }
   }
 
+  @Test
+  public void testPartitionsForIncrCompaction() throws Exception {
+    HoodieWriteConfig config = getConfigBuilder()
+        .withIncrementalTableServiceEnabled(true)
+        .withCompactionConfig(HoodieCompactionConfig
+            .newBuilder()
+            .withCompactionStrategy(new SmallBoundedIOCompactionStrategy())
+            .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+        .build();
+    String[] partitions = {"20250115"};
+
+    HoodieWriteConfig config2 = getConfigBuilder()
+        .withIncrementalTableServiceEnabled(true)
+        .withCompactionConfig(HoodieCompactionConfig
+            .newBuilder()
+            .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+        .build();
+    String[] partitions2 = {"20250116"};
+
+    try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
+      prepareRecords(writeClient, config, partitions);
+      prepareRecords(writeClient, config, partitions);
+      compact(writeClient, writeClient.createNewInstantTime());
+      HoodieCompactionPlan compactionPlan1 = getLatestCompactionPlan();
+
+      List<String> affectedPartitions = compactionPlan1.getOperations().stream()
+          .map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+      // compaction including 20250115
+      assertTrue(affectedPartitions.contains(partitions[0]));
+      List<String> missingPartitions = compactionPlan1.getMissingSchedulePartitions();
+      // compaction missing 20250115 because not all the fileSlices are all processed.
+      assertTrue(missingPartitions.contains(partitions[0]));
+    }
+
+    try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config2)) {
+      prepareRecords(writeClient, config2, partitions2);
+      prepareRecords(writeClient, config2, partitions2);
+      compact(writeClient, writeClient.createNewInstantTime());
+      HoodieCompactionPlan compactionPlan2 = getLatestCompactionPlan();
+      List<String> affectedPartitions2 = compactionPlan2.getOperations().stream()
+          .map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+      // compaction including 20250115 (fetched from recorded missing partitions)
+      assertTrue(affectedPartitions2.contains(partitions[0]));
+      // compaction including 20250116 (fetched from commit meta)
+      assertTrue(affectedPartitions2.contains(partitions2[0]));
+      // no missing partitions because all the fileSlices are all processed.
+      List<String> missingPartitions2 = compactionPlan2.getMissingSchedulePartitions();
+      assertTrue(missingPartitions2.isEmpty());
+    }
+  }
+
+  private void prepareRecords(SparkRDDWriteClient writeClient, HoodieWriteConfig config, String[] partitions) throws Exception {
+    initTestDataGenerator(partitions);
+    String newCommitTime = writeClient.createNewInstantTime();
+    writeClient.startCommitWithTime(newCommitTime);
+
+    // insert
+    List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
+    JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1);
+    writeClient.insert(recordsRDD, newCommitTime).collect();
+
+    // update
+    newCommitTime = writeClient.createNewInstantTime();
+    updateRecords(config, newCommitTime, records);
+  }
+
+  private HoodieCompactionPlan getLatestCompactionPlan() {
+    metaClient.reloadActiveTimeline();
+    Option<HoodieInstant> latestCompactionInstant = metaClient.getActiveTimeline().getCommitTimeline().lastInstant();
+    HoodieInstant compactionPlanInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION,
+        latestCompactionInstant.get().requestedTime(), InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+    Option<byte[]> details = metaClient.getActiveTimeline().readCompactionPlanAsBytes(compactionPlanInstant);
+
+    return CompactionUtils.getCompactionPlan(metaClient, details);
+  }
+
   @Override
   protected HoodieTableType getTableType() {
     return HoodieTableType.MERGE_ON_READ;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java
new file mode 100644
index 00000000000..06ca7f24819
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.table.action.compact.strategy;
+
+import org.apache.hudi.avro.model.HoodieCompactionOperation;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class SmallBoundedIOCompactionStrategy extends BoundedIOCompactionStrategy {
+
+  // Compare to BoundedIOCompactionStrategy, SmallBoundedIOCompactionStrategy is limit targetIO to 0.1M
+  private final double targetIO = 0.1;
+  private final long opIo = 1;
+  public SmallBoundedIOCompactionStrategy() {
+    super();
+  }
+
+  @Override
+  public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+                                                                            List<HoodieCompactionOperation> operations,
+                                                                            List<HoodieCompactionPlan> pendingCompactionPlans) {
+    ArrayList<String> missingPartitions = new ArrayList<>();
+    List<HoodieCompactionOperation> finalOperations = new ArrayList<>();
+    double targetIORemaining = targetIO;
+    for (HoodieCompactionOperation op : operations) {
+      if (targetIORemaining > 0) {
+        targetIORemaining -= opIo;
+        finalOperations.add(op);
+      } else if (writeConfig.isIncrementalTableServiceEnabled()) {
+        missingPartitions.add(op.getPartitionPath());
+      } else  {
+        return Pair.of(finalOperations, Collections.emptyList());
+      }
+    }
+    return Pair.of(finalOperations, missingPartitions);
+  }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
index 6a43205128e..ff508c2ad50 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
@@ -486,6 +486,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
     withTempDir { tmp =>
       val tableName = generateTableName
       val basePath = s"${tmp.getCanonicalPath}/$tableName"
+      val config = Map("hoodie.avro.schema.validate"-> "false")
 
       spark.sql(
         s"""
@@ -505,7 +506,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
            | location '$basePath'
      """.stripMargin)
 
-      writeRecords(2, 4, 0, basePath, Map("hoodie.avro.schema.validate"-> "false"))
+      writeRecords(2, 4, 0, basePath, config)
       val conf = new Configuration
       val metaClient = HoodieTestUtils.createMetaClient(new HadoopStorageConfiguration(conf), basePath)
       assert(0 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
@@ -516,16 +517,19 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
       assert(0 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
       assert(1 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
 
+      writeRecords(2, 4, 0, basePath, config)
       spark.sql(s"call run_clustering(table => '$tableName', op => 'execute')")
       metaClient.reloadActiveTimeline()
       assert(1 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
       assert(0 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
 
+      writeRecords(2, 4, 0, basePath, config)
       spark.sql(s"call run_clustering(table => '$tableName')")
       metaClient.reloadActiveTimeline()
       assert(2 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
       assert(0 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
 
+      writeRecords(2, 4, 0, basePath, config)
       spark.sql(s"call run_clustering(table => '$tableName')")
       metaClient.reloadActiveTimeline()
       assert(3 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
@@ -574,9 +578,10 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
 
       val fileNum = 20
       val numRecords = 400000
+      val config = metadataOpts ++ Map("hoodie.avro.schema.validate" -> "false")
 
       // insert records
-      writeRecords(fileNum, numRecords, 0, basePath,  metadataOpts ++ Map("hoodie.avro.schema.validate"-> "false"))
+      val records = writeRecords(fileNum, numRecords, 0, basePath, config)
       val conf = new Configuration
       val metaClient = HoodieTestUtils.createMetaClient(new HadoopStorageConfiguration(conf), basePath)
       val avgSize = avgRecord(metaClient.getActiveTimeline)
@@ -598,7 +603,11 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
       val orderAllFiles = fileIndex1.allBaseFiles.size
       val c2OrderFilterCount = fileIndex1.listFiles(Seq(), Seq(dataFilterC2)).head.files.size
       val c3OrderFilterCount = fileIndex1.listFiles(Seq(), Seq(dataFilterC3)).head.files.size
+      val fs = new Path(basePath).getFileSystem(spark.sessionState.newHadoopConf())
 
+      // re-create and ingestion + clustering
+      fs.delete(new Path(basePath), true)
+      writeRecordsArray(fileNum, records, basePath, config ++ Map("hoodie.table.name" -> tableName))
       spark.sql(
         s"""call run_clustering(table => '$tableName', order => 'c2,c3', order_strategy => 'z-order', options => "
            | hoodie.copyonwrite.record.size.estimate=$avgSize,
@@ -788,7 +797,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
     Math.ceil((1.0 * totalByteSize) / totalRecordsCount).toLong
   }
 
-  def writeRecords(files: Int, numRecords: Int, partitions: Int, location: String, options: Map[String, String]): Unit = {
+  def writeRecords(files: Int, numRecords: Int, partitions: Int, location: String, options: Map[String, String]): util.ArrayList[Row] = {
     val records = new util.ArrayList[Row](numRecords)
     val rowDimension = Math.ceil(Math.sqrt(numRecords)).toInt
 
@@ -811,6 +820,19 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
       StructField("c3", DataTypes.StringType, nullable = true, Metadata.empty)
     ))
 
+    // files can not effect for hudi
+    val df = spark.createDataFrame(records, struct).repartition(files)
+    writeDF(df, location, options)
+    records
+  }
+
+  def writeRecordsArray(files: Int, records: util.ArrayList[Row], location: String, options: Map[String, String]): Unit = {
+    val struct = StructType(Array[StructField](
+      StructField("c1", DataTypes.IntegerType, nullable = true, Metadata.empty),
+      StructField("c2", DataTypes.StringType, nullable = true, Metadata.empty),
+      StructField("c3", DataTypes.StringType, nullable = true, Metadata.empty)
+    ))
+
     // files can not effect for hudi
     val df = spark.createDataFrame(records, struct).repartition(files)
     writeDF(df, location, options)