You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by zh...@apache.org on 2025/01/23 01:45:00 UTC
(hudi) branch incremental-TableService-CI3 updated: HUDI-8900
This is an automated email from the ASF dual-hosted git repository.
zhangyue19921010 pushed a commit to branch incremental-TableService-CI3
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/incremental-TableService-CI3 by this push:
new 8eff45898bd HUDI-8900
8eff45898bd is described below
commit 8eff45898bdaf0dd748657ad6d7699d3da69c998
Author: YueZhang <zh...@163.com>
AuthorDate: Thu Jan 23 09:44:34 2025 +0800
HUDI-8900
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 16 ++
.../action/BaseTableServicePlanActionExecutor.java | 200 +++++++++++++
.../action/IncrementalPartitionAwareStrategy.java | 48 ++++
.../cluster/ClusteringPlanActionExecutor.java | 11 +-
.../cluster/ClusteringPlanPartitionFilter.java | 26 +-
...sistentHashingBucketClusteringPlanStrategy.java | 5 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 4 +-
.../PartitionAwareClusteringPlanStrategy.java | 96 +++++--
.../compact/ScheduleCompactionActionExecutor.java | 8 +-
.../BaseHoodieCompactionPlanGenerator.java | 24 +-
.../generators/HoodieCompactionPlanGenerator.java | 18 +-
.../HoodieLogCompactionPlanGenerator.java | 25 +-
.../strategy/BoundedIOCompactionStrategy.java | 21 +-
.../BoundedPartitionAwareCompactionStrategy.java | 17 +-
.../compact/strategy/CompactionStrategy.java | 29 +-
.../strategy/CompositeCompactionStrategy.java | 26 +-
.../strategy/DayBasedCompactionStrategy.java | 10 +-
.../LogFileNumBasedCompactionStrategy.java | 26 +-
.../LogFileSizeBasedCompactionStrategy.java | 29 +-
.../PartitionRegexBasedCompactionStrategy.java | 9 +-
.../strategy/UnBoundedCompactionStrategy.java | 9 +-
.../UnBoundedPartitionAwareCompactionStrategy.java | 14 +-
.../TestBaseTableServicePlanActionExecutor.java | 308 +++++++++++++++++++++
.../TestPartitionAwareClusteringPlanStrategy.java | 7 -
.../FlinkSizeBasedClusteringPlanStrategy.java | 3 +-
...inkSizeBasedClusteringPlanStrategyRecently.java | 8 +-
.../TestFlinkSizeBasedClusteringPlanStrategy.java | 4 +-
.../JavaSizeBasedClusteringPlanStrategy.java | 3 +-
.../strategy/SparkSingleFileSortPlanStrategy.java | 6 +-
.../SparkSizeBasedClusteringPlanStrategy.java | 3 +-
.../TestSparkSizeBasedClusteringPlanStrategy.java | 2 +-
.../table/action/cluster/ClusteringTestUtils.java | 13 +
.../action/cluster/TestIncrementalClustering.java | 218 +++++++++++++++
...TestSparkBuildClusteringGroupsForPartition.java | 6 +-
.../TestSparkClusteringPlanPartitionFilter.java | 10 +-
.../strategy/TestHoodieCompactionStrategy.java | 180 ++++++++----
.../src/main/avro/HoodieClusteringPlan.avsc | 10 +-
.../src/main/avro/HoodieCompactionOperation.avsc | 8 +
.../hudi/common/testutils/CompactionTestUtils.java | 2 +-
.../hudi/common/testutils/FileCreateUtils.java | 5 +
.../org/apache/hudi/utils/TestClusteringUtil.java | 2 +-
.../org/apache/hudi/utils/TestCompactionUtil.java | 2 +-
.../hudi/common/testutils/HoodieTestTable.java | 21 ++
.../table/action/compact/TestHoodieCompactor.java | 96 ++++++-
.../strategy/SmallBoundedIOCompactionStrategy.java | 59 ++++
.../hudi/procedure/TestClusteringProcedure.scala | 28 +-
46 files changed, 1466 insertions(+), 209 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index c7e14b6b4e1..0204f57c71e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -808,6 +808,13 @@ public class HoodieWriteConfig extends HoodieConfig {
.withDocumentation("Avro schema of the partial updates. This is automatically set by the "
+ "Hudi write client and user is not expected to manually change the value.");
+ public static final ConfigProperty<Boolean> INCREMENTAL_TABLE_SERVICE_ENABLED = ConfigProperty
+ .key("hoodie.table.services.incremental.enabled")
+ .defaultValue(true)
+ .markAdvanced()
+ .sinceVersion("1.0.0")
+ .withDocumentation("Whether to enable incremental table service. So far Clustering and Compaction support incremental processing.");
+
/**
* Config key with boolean value that indicates whether record being written during MERGE INTO Spark SQL
* operation are already prepped.
@@ -2179,6 +2186,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return getString(WRITE_PARTIAL_UPDATE_SCHEMA);
}
+ public boolean isIncrementalTableServiceEnabled() {
+ return getBoolean(INCREMENTAL_TABLE_SERVICE_ENABLED);
+ }
+
public double getParquetCompressionRatio() {
return getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION);
}
@@ -3341,6 +3352,11 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
+ public Builder withIncrementalTableServiceEnabled(boolean incrementalTableServiceEnabled) {
+ writeConfig.setValue(INCREMENTAL_TABLE_SERVICE_ENABLED, String.valueOf(incrementalTableServiceEnabled));
+ return this;
+ }
+
protected void setDefaults() {
writeConfig.setDefaultValue(MARKERS_TYPE, getDefaultMarkersType(engineType));
// Check for mandatory properties
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java
new file mode 100644
index 00000000000..06779a3d5e8
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseTableServicePlanActionExecutor.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
+import org.apache.hudi.common.util.ClusteringUtils;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.table.HoodieTable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
+
+public abstract class BaseTableServicePlanActionExecutor<T, I, K, O, R> extends BaseActionExecutor<T, I, K, O, R> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseTableServicePlanActionExecutor.class);
+ private static final Set<String> MOR_COMMITS = CollectionUtils.createSet(DELTA_COMMIT_ACTION, REPLACE_COMMIT_ACTION);
+ private static final Set<String> COW_COMMITS = CollectionUtils.createSet(COMMIT_ACTION, REPLACE_COMMIT_ACTION);
+
+ public BaseTableServicePlanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config,
+ HoodieTable<T, I, K, O> table, String instantTime) {
+ super(context, config, table, instantTime);
+ }
+
+ /**
+ * Get partitions, if strategy implement `IncrementalPartitionAwareStrategy` then return incremental partitions,
+ * otherwise return all partitions of the table
+ * @param strategy
+ * @return
+ */
+ public List<String> getPartitions(Object strategy, TableServiceType type) {
+ if (config.isIncrementalTableServiceEnabled() && strategy instanceof IncrementalPartitionAwareStrategy) {
+ try {
+ // get incremental partitions.
+ LOG.info("Start to fetch incremental partitions for " + type);
+ Pair<Option<HoodieInstant>, Set<String>> lastInstantAndIncrPartitions = getIncrementalPartitions(type);
+ Option<HoodieInstant> lastCompleteTableServiceInstant = lastInstantAndIncrPartitions.getLeft();
+ Set<String> incrementalPartitions = lastInstantAndIncrPartitions.getRight();
+
+ if (lastCompleteTableServiceInstant.isPresent()) {
+ if (!incrementalPartitions.isEmpty()) {
+ LOG.info("Fetched incremental partitions for " + type + ". " + incrementalPartitions + ". Instant " + instantTime);
+ return new ArrayList<>(incrementalPartitions);
+ } else {
+ // handle the case the writer just commits the empty commits continuously
+ // the incremental partition list is empty we just skip the scheduling
+ LOG.info("Incremental partitions are empty. Skip current schedule " + instantTime);
+ return Collections.emptyList();
+ }
+ }
+ // Last complete table service commit maybe archived.
+ // fall back to get all partitions.
+ LOG.info("No previous completed table service instant, fall back to get all partitions");
+ } catch (Exception ex) {
+ LOG.warn("Failed to get incremental partitions", ex);
+ }
+ }
+
+ // get all partitions
+ LOG.info("Start to fetch all partitions for " + type + ". Instant " + instantTime);
+ return FSUtils.getAllPartitionPaths(context, table.getMetaClient().getStorage(),
+ config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ }
+
+ public Pair<Option<HoodieInstant>, Set<String>> getIncrementalPartitions(TableServiceType type) {
+ Pair<Option<HoodieInstant>, List<String>> missingPair = fetchMissingPartitions(type);
+ Option<HoodieInstant> lastCompleteTableServiceInstant = missingPair.getLeft();
+ List<String> missingPartitions = missingPair.getRight();
+
+ String leftBoundary = lastCompleteTableServiceInstant.isPresent()
+ ? missingPair.getLeft().get().requestedTime() : HoodieTimeline.INIT_INSTANT_TS;
+ String rightBoundary = instantTime;
+ // compute [leftBoundary, rightBoundary) as time window
+ HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
+ Set<String> partitionsInCommitMeta = table.getActiveTimeline().filterCompletedInstants().getCommitsTimeline().getInstantsAsStream()
+ .filter(instant -> {
+ // ignore lastCompleteTableServiceInstant(left boundary) itself
+ return !(lastCompleteTableServiceInstant.isPresent() && instant.equals(lastCompleteTableServiceInstant.get()));
+ })
+ .filter(this::filterCommitByTableType).flatMap(instant -> {
+ try {
+ String completionTime = instant.getCompletionTime();
+ if (completionTime.compareTo(leftBoundary) >= 0 && completionTime.compareTo(rightBoundary) < 0) {
+ HoodieCommitMetadata metadata = TimelineUtils.getCommitMetadata(instant, activeTimeline);
+ return metadata.getWriteStats().stream().map(HoodieWriteStat::getPartitionPath);
+ }
+ return Stream.empty();
+ } catch (IOException e) {
+ throw new HoodieIOException("Failed to get commit meta " + instant, e);
+ }
+ }).collect(Collectors.toSet());
+
+ partitionsInCommitMeta.addAll(missingPartitions);
+ return Pair.of(lastCompleteTableServiceInstant, partitionsInCommitMeta);
+ }
+
+ private boolean filterCommitByTableType(HoodieInstant instant) {
+ switch (table.getMetaClient().getTableType()) {
+ case MERGE_ON_READ: {
+ // for mor only take care of delta commit and replace commit
+ return MOR_COMMITS.contains(instant.getAction());
+ }
+ case COPY_ON_WRITE: {
+ // for mor only take care of commit and replace commit
+ return COW_COMMITS.contains(instant.getAction());
+ }
+ default:
+ throw new HoodieException("Un-supported table type " + table.getMetaClient().getTableType());
+ }
+ }
+
+ public Pair<Option<HoodieInstant>, List<String>> fetchMissingPartitions(TableServiceType tableServiceType) {
+ if (!config.isIncrementalTableServiceEnabled()) {
+ return Pair.of(Option.empty(), Collections.emptyList());
+ }
+
+ Option<HoodieInstant> instant = Option.empty();
+ List<String> missingPartitions = new ArrayList<>();
+
+ switch (tableServiceType) {
+ case COMPACT:
+ case LOG_COMPACT: {
+ Option<HoodieInstant> lastCompactionCommitInstant = table.getActiveTimeline()
+ .filterCompletedInstants().getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION)).lastInstant();
+ if (lastCompactionCommitInstant.isPresent()) {
+ instant = lastCompactionCommitInstant;
+ String action = tableServiceType.equals(TableServiceType.COMPACT) ? HoodieTimeline.COMPACTION_ACTION : HoodieTimeline.LOG_COMPACTION_ACTION;
+ HoodieInstant compactionPlanInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, action,
+ instant.get().requestedTime(), InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+ Option<byte[]> details = table.getMetaClient().getActiveTimeline().readCompactionPlanAsBytes(compactionPlanInstant);
+ HoodieCompactionPlan compactionPlan = CompactionUtils.getCompactionPlan(table.getMetaClient(), details);
+ if (compactionPlan.getMissingSchedulePartitions() != null) {
+ missingPartitions = compactionPlan.getMissingSchedulePartitions();
+ }
+ }
+ break;
+ }
+ case CLUSTER: {
+ Option<HoodieInstant> lastClusteringInstant = table.getActiveTimeline().filterCompletedInstants().getLastClusteringInstant();
+ if (lastClusteringInstant.isPresent()) {
+ instant = lastClusteringInstant;
+ Option<Pair<HoodieInstant, HoodieClusteringPlan>> clusteringPlan = ClusteringUtils.getClusteringPlan(table.getMetaClient(), lastClusteringInstant.get());
+ if (clusteringPlan.isPresent() && clusteringPlan.get().getRight().getMissingSchedulePartitions() != null) {
+ missingPartitions = clusteringPlan.get().getRight().getMissingSchedulePartitions();
+ }
+ }
+ break;
+ }
+ default:
+ throw new HoodieException("Un-supported incremental table service " + tableServiceType);
+ }
+
+ return Pair.of(instant, missingPartitions);
+ }
+
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java
new file mode 100644
index 00000000000..717d508f7ae
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/IncrementalPartitionAwareStrategy.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action;
+
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.List;
+
+/**
+ * Marking interface for table service strategy that utilize incremental partitions.
+ *
+ * <p> Any strategy class that implements this `IncrementalPartitionAwareStrategy` could have the ability to perform incremental partitions processing.
+ * Currently, Incremental partitions will be passed to the strategy instance as a best-effort. In the following cases, the partitions would fallback to full partition list:
+ *
+ * <ul>
+ * <li> Executing Table Service for the first time. </li>
+ * <li> The last completed table service instant is archived. </li>
+ * <li> Any exception thrown during retrieval of incremental partitions. </li>
+ * </ul>
+ */
+public interface IncrementalPartitionAwareStrategy {
+
+ /**
+ * Filter the given incremental partitions.
+ * @param writeConfig
+ * @param partitions
+ * @return Pair of final processing partition paths and filtered partitions which will be recorded as missing partitions.
+ * Different strategies can individually implement whether to record, or which partitions to record as missing partitions.
+ */
+ Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions);
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
index d698053d7b5..a92f8c87ba5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
@@ -21,6 +21,7 @@ package org.apache.hudi.table.action.cluster;
import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.TableServiceType;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -31,17 +32,19 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.BaseActionExecutor;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
import org.apache.hudi.table.action.cluster.strategy.ClusteringPlanStrategy;
+import org.apache.hudi.util.Lazy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
-public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, O, Option<HoodieClusteringPlan>> {
+public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseTableServicePlanActionExecutor<T, I, K, O, Option<HoodieClusteringPlan>> {
private static final Logger LOG = LoggerFactory.getLogger(ClusteringPlanActionExecutor.class);
@@ -84,7 +87,9 @@ public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseActionExecutor
ClusteringPlanStrategy.checkAndGetClusteringPlanStrategy(config),
new Class<?>[] {HoodieTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, table, context, config);
- return strategy.generateClusteringPlan();
+ Lazy<List<String>> partitions = Lazy.lazily(() -> getPartitions(strategy, TableServiceType.CLUSTER));
+
+ return strategy.generateClusteringPlan(this, partitions);
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
index ecc3706f674..47c483e265b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilter.java
@@ -18,12 +18,14 @@
package org.apache.hudi.table.action.cluster;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieClusteringException;
import org.joda.time.DateTime;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@@ -42,11 +44,11 @@ import java.util.stream.Stream;
*/
public class ClusteringPlanPartitionFilter {
- public static List<String> filter(List<String> partitions, HoodieWriteConfig config) {
+ public static Pair<List<String>, List<String>> filter(List<String> partitions, HoodieWriteConfig config) {
ClusteringPlanPartitionFilterMode mode = config.getClusteringPlanPartitionFilterMode();
switch (mode) {
case NONE:
- return partitions;
+ return Pair.of(partitions, Collections.emptyList());
case RECENT_DAYS:
return recentDaysFilter(partitions, config);
case SELECTED_PARTITIONS:
@@ -58,7 +60,7 @@ public class ClusteringPlanPartitionFilter {
}
}
- private static List<String> dayRollingFilter(List<String> partitions, HoodieWriteConfig config) {
+ private static Pair<List<String>, List<String>> dayRollingFilter(List<String> partitions, HoodieWriteConfig config) {
int hour = DateTime.now().getHourOfDay();
int len = partitions.size();
List<String> selectPt = new ArrayList<>();
@@ -68,20 +70,22 @@ public class ClusteringPlanPartitionFilter {
selectPt.add(partitions.get(i));
}
}
- return selectPt;
+ return Pair.of(selectPt, Collections.emptyList());
}
- private static List<String> recentDaysFilter(List<String> partitions, HoodieWriteConfig config) {
+ private static Pair<List<String>, List<String>> recentDaysFilter(List<String> partitions, HoodieWriteConfig config) {
int targetPartitionsForClustering = config.getTargetPartitionsForClustering();
int skipPartitionsFromLatestForClustering = config.getSkipPartitionsFromLatestForClustering();
- return partitions.stream()
- .sorted(Comparator.reverseOrder())
- .skip(Math.max(skipPartitionsFromLatestForClustering, 0))
+ int skipOffset = Math.max(skipPartitionsFromLatestForClustering, 0);
+ List<String> sortedPartitions = partitions.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList());
+ List<String> missingPartitions = sortedPartitions.subList(0, skipOffset);
+ return Pair.of(sortedPartitions.stream()
+ .skip(skipOffset)
.limit(targetPartitionsForClustering > 0 ? targetPartitionsForClustering : partitions.size())
- .collect(Collectors.toList());
+ .collect(Collectors.toList()), missingPartitions);
}
- private static List<String> selectedPartitionsFilter(List<String> partitions, HoodieWriteConfig config) {
+ private static Pair<List<String>, List<String>> selectedPartitionsFilter(List<String> partitions, HoodieWriteConfig config) {
Stream<String> filteredPartitions = partitions.stream();
String beginPartition = config.getBeginPartitionForClustering();
@@ -94,6 +98,6 @@ public class ClusteringPlanPartitionFilter {
filteredPartitions = filteredPartitions.filter(path -> path.compareTo(endPartition) <= 0);
}
- return filteredPartitions.collect(Collectors.toList());
+ return Pair.of(filteredPartitions.collect(Collectors.toList()), Collections.emptyList());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
index 27fea59fa9f..095066d73e7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/BaseConsistentHashingBucketClusteringPlanStrategy.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.util.collection.Triple;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -95,7 +96,7 @@ public abstract class BaseConsistentHashingBucketClusteringPlanStrategy<T extend
* Generate cluster group based on split, merge and sort rules
*/
@Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
Option<HoodieConsistentHashingMetadata> metadata = ConsistentBucketIndexUtils.loadMetadata(getHoodieTable(), partitionPath);
ValidationUtils.checkArgument(metadata.isPresent(), "Metadata is empty for partition: " + partitionPath);
ConsistentBucketIdentifier identifier = new ConsistentBucketIdentifier(metadata.get());
@@ -128,7 +129,7 @@ public abstract class BaseConsistentHashingBucketClusteringPlanStrategy<T extend
.build();
}).collect(Collectors.toList()));
}
- return ret.stream();
+ return Pair.of(ret.stream(), true);
}
/**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
index a6894388f6d..084beee85f1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
@@ -32,7 +32,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
+import org.apache.hudi.util.Lazy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +103,7 @@ public abstract class ClusteringPlanStrategy<T,I,K,O> implements Serializable {
*
* If there is no data available to cluster, return None.
*/
- public abstract Option<HoodieClusteringPlan> generateClusteringPlan();
+ public abstract Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions);
/**
* Check if the clustering can proceed. If not (i.e., return false), the PlanStrategy will generate an empty plan to stop the scheduling.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
index 515d6532871..3f0a97eff22 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -22,22 +22,27 @@ import org.apache.hudi.avro.model.HoodieClusteringGroup;
import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieClusteringStrategy;
import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.TableServiceType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilter;
+import org.apache.hudi.util.Lazy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -45,7 +50,7 @@ import java.util.stream.Stream;
/**
* Scheduling strategy with restriction that clustering groups can only contain files from same partition.
*/
-public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends ClusteringPlanStrategy<T,I,K,O> {
+public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends ClusteringPlanStrategy<T,I,K,O> implements IncrementalPartitionAwareStrategy {
private static final Logger LOG = LoggerFactory.getLogger(PartitionAwareClusteringPlanStrategy.class);
public PartitionAwareClusteringPlanStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
@@ -54,8 +59,10 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
/**
* Create Clustering group based on files eligible for clustering in the partition.
+ * return stream of HoodieClusteringGroup and boolean partial Scheduled indicating whether all given fileSlices in the current partition have been processed.
+ * For example, if some file slices will not be processed due to writeConfig.getClusteringMaxNumGroups(), then return false
*/
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
HoodieWriteConfig writeConfig = getWriteConfig();
List<Pair<List<FileSlice>, Integer>> fileSliceGroups = new ArrayList<>();
@@ -68,6 +75,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
- (o1.getBaseFile().isPresent() ? o1.getBaseFile().get().getFileSize() : writeConfig.getParquetMaxFileSize())));
long totalSizeSoFar = 0;
+ boolean partialScheduled = false;
for (FileSlice currentSlice : sortedFileSlices) {
long currentSize = currentSlice.getBaseFile().isPresent() ? currentSlice.getBaseFile().get().getFileSize() : writeConfig.getParquetMaxFileSize();
@@ -83,6 +91,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
// if fileSliceGroups's size reach the max group, stop loop
if (fileSliceGroups.size() >= writeConfig.getClusteringMaxNumGroups()) {
LOG.info("Having generated the maximum number of groups : " + writeConfig.getClusteringMaxNumGroups());
+ partialScheduled = true;
break;
}
}
@@ -102,25 +111,23 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
}
}
- return fileSliceGroups.stream().map(fileSliceGroup ->
+ return Pair.of(fileSliceGroups.stream().map(fileSliceGroup ->
HoodieClusteringGroup.newBuilder()
.setSlices(getFileSliceInfo(fileSliceGroup.getLeft()))
.setNumOutputFileGroups(fileSliceGroup.getRight())
.setMetrics(buildMetrics(fileSliceGroup.getLeft()))
- .build());
+ .build()), partialScheduled);
}
/**
* Return list of partition paths to be considered for clustering.
*/
- protected List<String> filterPartitionPaths(List<String> partitionPaths) {
- List<String> filteredPartitions = ClusteringPlanPartitionFilter.filter(partitionPaths, getWriteConfig());
- LOG.debug("Filtered to the following partitions: " + filteredPartitions);
- return filteredPartitions;
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions) {
+ return ClusteringPlanPartitionFilter.filter(partitions, getWriteConfig());
}
@Override
- public Option<HoodieClusteringPlan> generateClusteringPlan() {
+ public Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions) {
if (!checkPrecondition()) {
return Option.empty();
}
@@ -131,36 +138,74 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
String partitionSelected = config.getClusteringPartitionSelected();
LOG.info("Scheduling clustering partitionSelected: {}", partitionSelected);
- List<String> partitionPaths;
+ List<String> partitionPaths = new ArrayList<>();
+ List<String> missingPartitions = new ArrayList<>();
+ List<HoodieClusteringGroup> clusteringGroups;
+ int clusteringMaxNumGroups = getWriteConfig().getClusteringMaxNumGroups();
if (StringUtils.isNullOrEmpty(partitionSelected)) {
// get matched partitions if set
- partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(
- getEngineContext(), metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath()));
+ partitionPaths = getRegexPatternMatchedPartitions(config, partitions.get());
// filter the partition paths if needed to reduce list status
} else {
partitionPaths = Arrays.asList(partitionSelected.split(","));
+ // Users may temporarily set specific partitions for clustering.
+ // Ensure the coherence of the missing partitions.
+ missingPartitions = (List<String>)executor.fetchMissingPartitions(TableServiceType.CLUSTER).getRight();
}
- partitionPaths = filterPartitionPaths(partitionPaths);
+ Pair<List<String>, List<String>> partitionsPair = filterPartitionPaths(getWriteConfig(), partitionPaths);
+ partitionPaths = partitionsPair.getLeft();
+ missingPartitions.addAll(partitionsPair.getRight());
LOG.info("Scheduling clustering partitionPaths: {}", partitionPaths);
+ LOG.info("Missing Scheduled clustering partitionPaths: {}", missingPartitions);
if (partitionPaths.isEmpty()) {
// In case no partitions could be picked, return no clustering plan
return Option.empty();
}
- List<HoodieClusteringGroup> clusteringGroups = getEngineContext()
- .flatMap(
- partitionPaths,
- partitionPath -> {
- List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
- return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).limit(getWriteConfig().getClusteringMaxNumGroups());
- },
- partitionPaths.size())
- .stream()
- .limit(getWriteConfig().getClusteringMaxNumGroups())
- .collect(Collectors.toList());
+ List<Pair<List<HoodieClusteringGroup>, String>> res = getEngineContext().map(partitionPaths, partitionPath -> {
+ List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
+ Pair<Stream<HoodieClusteringGroup>, Boolean> groupPair = buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible);
+ List<HoodieClusteringGroup> clusteringGroupsPartition = groupPair.getLeft().collect(Collectors.toList());
+ boolean partialScheduled = groupPair.getRight();
+ // return missed partition path
+ // because the candidate fileSlices in the current partition have not been completely processed.
+ if (clusteringGroupsPartition.size() > clusteringMaxNumGroups) {
+ return Pair.of(clusteringGroupsPartition.subList(0, clusteringMaxNumGroups), partitionPath);
+ } else if (partialScheduled) {
+ return Pair.of(clusteringGroupsPartition, partitionPath);
+ } else {
+ return Pair.of(clusteringGroupsPartition, "");
+ }
+ }, partitionPaths.size());
+
+ if (config.isIncrementalTableServiceEnabled()) {
+ Set<String> skippedPartitions = new HashSet<>();
+ List<HoodieClusteringGroup> collectedGroups = res.stream().flatMap(pair -> {
+ String missingPartition = pair.getRight();
+ if (!StringUtils.isNullOrEmpty(missingPartition)) {
+ // missingPartition value is not empty, which means it related candidate fileSlices all not all processed.
+ // so that we need to mark this kind of partition as missing partition.
+ skippedPartitions.add(missingPartition);
+ }
+ return pair.getLeft().stream();
+ }).collect(Collectors.toList());
+
+ clusteringGroups = collectedGroups.stream().limit(clusteringMaxNumGroups).collect(Collectors.toList());
+ // sublist of collectedGroups are skipped, marking related partition as missing partitions.
+ collectedGroups.subList(Math.min(clusteringMaxNumGroups, collectedGroups.size()), collectedGroups.size()).forEach(group -> {
+ String missed = group.getSlices().get(0).getPartitionPath();
+ skippedPartitions.add(missed);
+ });
+
+ skippedPartitions.addAll(missingPartitions);
+ missingPartitions = new ArrayList<>(skippedPartitions);
+ } else {
+ clusteringGroups = res.stream().flatMap(pair -> pair.getLeft().stream()).limit(clusteringMaxNumGroups).collect(Collectors.toList());
+ missingPartitions = null;
+ }
if (clusteringGroups.isEmpty()) {
LOG.warn("No data available to cluster");
@@ -178,6 +223,7 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
.setExtraMetadata(getExtraMetadata())
.setVersion(getPlanVersion())
.setPreserveHoodieMetadata(true)
+ .setMissingSchedulePartitions(missingPartitions)
.build());
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
index e83800e45da..bd97d40f7fc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/ScheduleCompactionActionExecutor.java
@@ -34,7 +34,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCompactionException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.action.BaseActionExecutor;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
import org.apache.hudi.table.action.compact.plan.generators.BaseHoodieCompactionPlanGenerator;
import org.apache.hudi.table.action.compact.plan.generators.HoodieCompactionPlanGenerator;
import org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator;
@@ -50,7 +50,7 @@ import java.util.Map;
import static org.apache.hudi.common.util.CollectionUtils.nonEmpty;
import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
-public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, O, Option<HoodieCompactionPlan>> {
+public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseTableServicePlanActionExecutor<T, I, K, O, Option<HoodieCompactionPlan>> {
private static final Logger LOG = LoggerFactory.getLogger(ScheduleCompactionActionExecutor.class);
private WriteOperationType operationType;
@@ -73,9 +73,9 @@ public class ScheduleCompactionActionExecutor<T, I, K, O> extends BaseActionExec
private void initPlanGenerator(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table) {
if (WriteOperationType.COMPACT.equals(operationType)) {
- planGenerator = new HoodieCompactionPlanGenerator(table, context, config);
+ planGenerator = new HoodieCompactionPlanGenerator(table, context, config, this);
} else {
- planGenerator = new HoodieLogCompactionPlanGenerator(table, context, config);
+ planGenerator = new HoodieLogCompactionPlanGenerator(table, context, config, this);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
index 1884f4c5d40..0d392bc5ff9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
@@ -22,7 +22,6 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.data.HoodieAccumulator;
import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -41,6 +40,7 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
import org.apache.hudi.table.action.compact.CompactHelpers;
import org.slf4j.Logger;
@@ -64,11 +64,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
protected final HoodieTable<T, I, K, O> hoodieTable;
protected final HoodieWriteConfig writeConfig;
protected final transient HoodieEngineContext engineContext;
+ protected final BaseTableServicePlanActionExecutor executor;
- public BaseHoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
+ public BaseHoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+ BaseTableServicePlanActionExecutor executor) {
this.hoodieTable = table;
this.writeConfig = writeConfig;
this.engineContext = engineContext;
+ this.executor = executor;
}
@Nullable
@@ -82,13 +85,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
// TODO - rollback any compactions in flight
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
CompletionTimeQueryView completionTimeQueryView = metaClient.getTimelineLayout().getTimelineFactory().createCompletionTimeQueryView(metaClient);
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(
- engineContext, metaClient.getStorage(), writeConfig.getMetadataConfig(), metaClient.getBasePath());
+ List<String> partitionPaths = getPartitions();
int allPartitionSize = partitionPaths.size();
// filter the partition paths if needed to reduce list status
- partitionPaths = filterPartitionPathsByStrategy(partitionPaths);
+ Pair<List<String>, List<String>> partitionPair = filterPartitionPathsByStrategy(partitionPaths);
+ partitionPaths = partitionPair.getLeft();
+
LOG.info("Strategy: {} matched {} partition paths from all {} partitions for table {}",
writeConfig.getCompactionStrategy().getClass().getSimpleName(), partitionPaths.size(), allPartitionSize,
hoodieTable.getConfig().getBasePath());
@@ -169,7 +173,7 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
}
// Filter the compactions with the passed in filter. This lets us choose most effective compactions only
- HoodieCompactionPlan compactionPlan = getCompactionPlan(metaClient, operations);
+ HoodieCompactionPlan compactionPlan = getCompactionPlan(metaClient, operations, partitionPair);
ValidationUtils.checkArgument(
compactionPlan.getOperations().stream().noneMatch(
op -> fgIdsInPendingCompactionAndClustering.contains(new HoodieFileGroupId(op.getPartitionPath(), op.getFileId()))),
@@ -182,12 +186,14 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
return compactionPlan;
}
- protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations);
+ protected abstract List<String> getPartitions();
+
+ protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>,List<String>> partitionPair);
protected abstract boolean filterLogCompactionOperations();
- protected List<String> filterPartitionPathsByStrategy(List<String> partitionPaths) {
- return partitionPaths;
+ protected Pair<List<String>, List<String>> filterPartitionPathsByStrategy(List<String> partitionPaths) {
+ return Pair.of(partitionPaths, Collections.emptyList());
}
protected boolean filterFileSlice(FileSlice fileSlice, String lastCompletedInstantTime,
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
index a93ece710b0..378461f1b98 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieCompactionPlanGenerator.java
@@ -22,11 +22,13 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.TableServiceType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
import org.slf4j.Logger;
@@ -43,22 +45,28 @@ public class HoodieCompactionPlanGenerator<T extends HoodieRecordPayload, I, K,
private final CompactionStrategy compactionStrategy;
- public HoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
- super(table, engineContext, writeConfig);
+ public HoodieCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+ BaseTableServicePlanActionExecutor executor) {
+ super(table, engineContext, writeConfig, executor);
this.compactionStrategy = writeConfig.getCompactionStrategy();
LOG.info("Compaction Strategy used is: " + compactionStrategy.toString());
}
@Override
- protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations) {
+ protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>, List<String>> partitionPair) {
// Filter the compactions with the passed in filter. This lets us choose most effective
// compactions only
return compactionStrategy.generateCompactionPlan(writeConfig, operations,
- CompactionUtils.getAllPendingCompactionPlans(metaClient).stream().map(Pair::getValue).collect(toList()));
+ CompactionUtils.getAllPendingCompactionPlans(metaClient).stream().map(Pair::getValue).collect(toList()), getStrategyParams(), partitionPair);
}
@Override
- protected List<String> filterPartitionPathsByStrategy(List<String> partitionPaths) {
+ protected List<String> getPartitions() {
+ return executor.getPartitions(compactionStrategy, TableServiceType.COMPACT);
+ }
+
+ @Override
+ protected Pair<List<String>, List<String>> filterPartitionPathsByStrategy(List<String> partitionPaths) {
return compactionStrategy.filterPartitionPaths(writeConfig, partitionPaths);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
index a81ee663fa9..5d9337ac4e7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
@@ -26,13 +26,16 @@ import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.TableServiceType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner;
import org.apache.hudi.common.table.log.InstantRange;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
import org.apache.hudi.table.action.compact.LogCompactionExecutionHelper;
import org.slf4j.Logger;
@@ -45,25 +48,33 @@ import java.util.stream.Collectors;
public class HoodieLogCompactionPlanGenerator<T extends HoodieRecordPayload, I, K, O> extends BaseHoodieCompactionPlanGenerator<T, I, K, O> {
private static final Logger LOG = LoggerFactory.getLogger(HoodieLogCompactionPlanGenerator.class);
+ private final HoodieCompactionStrategy compactionStrategy;
- public HoodieLogCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
- super(table, engineContext, writeConfig);
- }
-
- @Override
- protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations) {
- HoodieCompactionStrategy compactionStrategy = HoodieCompactionStrategy.newBuilder()
+ public HoodieLogCompactionPlanGenerator(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig,
+ BaseTableServicePlanActionExecutor executor) {
+ super(table, engineContext, writeConfig, executor);
+ this.compactionStrategy = HoodieCompactionStrategy.newBuilder()
.setStrategyParams(getStrategyParams())
.setCompactorClassName(LogCompactionExecutionHelper.class.getName())
.build();
+ }
+
+ @Override
+ protected HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>, List<String>> partitionPair) {
return HoodieCompactionPlan.newBuilder()
.setOperations(operations)
.setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION)
.setStrategy(compactionStrategy)
+ .setMissingSchedulePartitions(partitionPair.getRight())
.setPreserveHoodieMetadata(true)
.build();
}
+ @Override
+ protected List<String> getPartitions() {
+ return executor.getPartitions(compactionStrategy, TableServiceType.LOG_COMPACT);
+ }
+
@Override
protected boolean filterFileSlice(FileSlice fileSlice, String lastCompletedInstantTime,
Set<HoodieFileGroupId> pendingFileGroupIds, Option<InstantRange> instantRange) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
index d93a50fd673..c2bd139fe9e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedIOCompactionStrategy.java
@@ -20,9 +20,11 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
/**
@@ -34,8 +36,10 @@ import java.util.List;
public class BoundedIOCompactionStrategy extends CompactionStrategy {
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
- List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+ List<HoodieCompactionOperation> operations,
+ List<HoodieCompactionPlan> pendingCompactionPlans) {
+ ArrayList<String> missingPartitions = new ArrayList<>();
// Iterate through the operations in order and accept operations as long as we are within the
// IO limit
// Preserves the original ordering of compactions
@@ -43,12 +47,15 @@ public class BoundedIOCompactionStrategy extends CompactionStrategy {
long targetIORemaining = writeConfig.getTargetIOPerCompactionInMB();
for (HoodieCompactionOperation op : operations) {
long opIo = op.getMetrics().get(TOTAL_IO_MB).longValue();
- targetIORemaining -= opIo;
- finalOperations.add(op);
- if (targetIORemaining <= 0) {
- return finalOperations;
+ if (targetIORemaining > 0) {
+ targetIORemaining -= opIo;
+ finalOperations.add(op);
+ } else if (writeConfig.isIncrementalTableServiceEnabled()) {
+ missingPartitions.add(op.getPartitionPath());
+ } else {
+ return Pair.of(finalOperations, Collections.emptyList());
}
}
- return finalOperations;
+ return Pair.of(finalOperations, missingPartitions);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
index 09c19b1aabe..3944a7658a2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
@@ -20,10 +20,12 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
@@ -45,28 +47,33 @@ public class BoundedPartitionAwareCompactionStrategy extends DayBasedCompactionS
ThreadLocal.withInitial(() -> new SimpleDateFormat(DayBasedCompactionStrategy.DATE_PARTITION_FORMAT));
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
// The earliest partition to compact - current day minus the target partitions limit
String earliestPartitionPathToCompact =
dateFormat.get().format(getDateAtOffsetFromToday(-1 * writeConfig.getTargetPartitionsPerDayBasedCompaction()));
// Filter out all partitions greater than earliestPartitionPathToCompact
-
- return operations.stream().collect(Collectors.groupingBy(HoodieCompactionOperation::getPartitionPath)).entrySet()
+ List<HoodieCompactionOperation> res = operations.stream().collect(Collectors.groupingBy(HoodieCompactionOperation::getPartitionPath)).entrySet()
.stream().sorted(Map.Entry.comparingByKey(DayBasedCompactionStrategy.comparator))
.filter(e -> DayBasedCompactionStrategy.comparator.compare(earliestPartitionPathToCompact, e.getKey()) >= 0)
.flatMap(e -> e.getValue().stream()).collect(Collectors.toList());
+
+ return Pair.of(res, Collections.emptyList());
}
+ /**
+ * For BoundedPartitionAwareCompactionStrategy, we will not record rolled partitions as missing partitions.
+ */
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
// The earliest partition to compact - current day minus the target partitions limit
String earliestPartitionPathToCompact =
dateFormat.get().format(getDateAtOffsetFromToday(-1 * writeConfig.getTargetPartitionsPerDayBasedCompaction()));
// Get all partitions and sort them
- return partitionPaths.stream().map(partition -> partition.replace("/", "-"))
+ List<String> res = partitionPaths.stream().map(partition -> partition.replace("/", "-"))
.sorted(Comparator.reverseOrder()).map(partitionPath -> partitionPath.replace("-", "/"))
.filter(e -> DayBasedCompactionStrategy.comparator.compare(earliestPartitionPathToCompact, e) >= 0).collect(Collectors.toList());
+ return Pair.of(res, Collections.emptyList());
}
public static Date getDateAtOffsetFromToday(int offset) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
index 5b2dba7ab7a..718dc685749 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
@@ -23,13 +23,16 @@ import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.client.utils.FileSliceMetricUtils;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
/**
* Strategy for compaction. Pluggable implementation to define how compaction should be done. The over-ridden
@@ -37,7 +40,7 @@ import java.util.Map;
* compaction operation to run in a single compaction. Implementation of CompactionStrategy cannot hold any state.
* Difference instantiations can be passed in every time
*/
-public abstract class CompactionStrategy implements Serializable {
+public abstract class CompactionStrategy implements IncrementalPartitionAwareStrategy, Serializable {
public static final String TOTAL_IO_READ_MB = "TOTAL_IO_READ_MB";
public static final String TOTAL_IO_WRITE_MB = "TOTAL_IO_WRITE_MB";
@@ -71,11 +74,18 @@ public abstract class CompactionStrategy implements Serializable {
* @return Compaction plan to be scheduled.
*/
public HoodieCompactionPlan generateCompactionPlan(HoodieWriteConfig writeConfig,
- List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+ List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans, Map<String, String> params, Pair<List<String>, List<String>> partitionPair) {
// Strategy implementation can overload this method to set specific compactor-id
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = orderAndFilter(writeConfig, operations, pendingCompactionPlans);
+ List<HoodieCompactionOperation> operationsToProcess = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ missingPartitions.addAll(partitionPair.getRight());
+ List<String> res = writeConfig.isIncrementalTableServiceEnabled() ? missingPartitions.stream().distinct().collect(Collectors.toList()) : null;
return HoodieCompactionPlan.newBuilder()
- .setOperations(orderAndFilter(writeConfig, operations, pendingCompactionPlans))
- .setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION).build();
+ .setOperations(operationsToProcess)
+ .setMissingSchedulePartitions(res)
+ .setVersion(CompactionUtils.LATEST_COMPACTION_METADATA_VERSION)
+ .build();
}
/**
@@ -85,11 +95,12 @@ public abstract class CompactionStrategy implements Serializable {
* @param writeConfig config for this compaction is passed in
* @param operations list of compactions collected
* @param pendingCompactionPlans Pending Compaction Plans for strategy to schedule next compaction plan
- * @return list of compactions to perform in this run
+ * @return Pair of list of compactions to perform in this run and missing partitions.
+ * Filtered out operations related partitions are missing partitions.
*/
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
- return operations;
+ return Pair.of(operations, Collections.emptyList());
}
/**
@@ -99,7 +110,7 @@ public abstract class CompactionStrategy implements Serializable {
* @param allPartitionPaths
* @return
*/
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
- return allPartitionPaths;
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ return Pair.of(allPartitionPaths, Collections.emptyList());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
index da90269509d..882f880069e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompositeCompactionStrategy.java
@@ -21,8 +21,11 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
/**
@@ -39,21 +42,30 @@ public class CompositeCompactionStrategy extends CompactionStrategy {
}
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig, List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+ List<HoodieCompactionOperation> operations,
+ List<HoodieCompactionPlan> pendingCompactionPlans) {
List<HoodieCompactionOperation> finalOperations = operations;
+ List<String> missingPartitions = new ArrayList<>();
for (CompactionStrategy strategy : strategies) {
- finalOperations = strategy.orderAndFilter(writeConfig, finalOperations, pendingCompactionPlans);
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = strategy.orderAndFilter(writeConfig, finalOperations, pendingCompactionPlans);
+ finalOperations = resPair.getLeft();
+ missingPartitions.addAll(resPair.getRight());
}
- return finalOperations;
+ return Pair.of(finalOperations, missingPartitions);
}
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
- List<String> finalPartitionPaths = allPartitionPaths;
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ List<String> partitionsToProcess = allPartitionPaths;
+ List<String> missingPartitions = new ArrayList<>();
+
for (CompactionStrategy strategy : strategies) {
- finalPartitionPaths = strategy.filterPartitionPaths(writeConfig, finalPartitionPaths);
+ Pair<List<String>, List<String>> innerRes = strategy.filterPartitionPaths(writeConfig, partitionsToProcess);
+ partitionsToProcess = innerRes.getLeft();
+ missingPartitions.addAll(innerRes.getRight());
}
- return finalPartitionPaths;
+ return writeConfig.isIncrementalTableServiceEnabled() ? Pair.of(partitionsToProcess, missingPartitions) : Pair.of(partitionsToProcess, Collections.emptyList());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
index d77d22d05c5..c7344c4590b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/DayBasedCompactionStrategy.java
@@ -18,11 +18,13 @@
package org.apache.hudi.table.action.compact.strategy;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
@@ -60,11 +62,15 @@ public class DayBasedCompactionStrategy extends BoundedIOCompactionStrategy {
return comparator;
}
+ /**
+ * For DayBasedCompactionStrategy, we will not record rolled partitions as missing partitions.
+ */
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
- return allPartitionPaths.stream().sorted(comparator)
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ List<String> res = allPartitionPaths.stream().sorted(comparator)
.collect(Collectors.toList()).subList(0, Math.min(allPartitionPaths.size(),
writeConfig.getTargetPartitionsPerDayBasedCompaction()));
+ return Pair.of(res, Collections.emptyList());
}
/**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
index 6f79b684d0a..efacfe8b650 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileNumBasedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@@ -34,12 +36,30 @@ public class LogFileNumBasedCompactionStrategy extends BoundedIOCompactionStrate
implements Comparator<HoodieCompactionOperation> {
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig, List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+ List<HoodieCompactionOperation> operations,
+ List<HoodieCompactionPlan> pendingCompactionPlans) {
Long numThreshold = writeConfig.getCompactionLogFileNumThreshold();
+ ArrayList<String> missingPartitions = new ArrayList<>();
+ boolean incrementalTableServiceEnabled = writeConfig.isIncrementalTableServiceEnabled();
List<HoodieCompactionOperation> filterOperator = operations.stream()
- .filter(e -> e.getDeltaFilePaths().size() >= numThreshold)
+ .filter(e -> {
+ if (incrementalTableServiceEnabled && e.getDeltaFilePaths().size() < numThreshold) {
+ missingPartitions.add(e.getPartitionPath());
+ }
+ return e.getDeltaFilePaths().size() >= numThreshold;
+ })
.sorted(this).collect(Collectors.toList());
- return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+
+ if (incrementalTableServiceEnabled) {
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+ List<HoodieCompactionOperation> compactOperations = resPair.getLeft();
+ List<String> innerMissingPartitions = resPair.getRight();
+ missingPartitions.addAll(innerMissingPartitions);
+ return Pair.of(compactOperations, missingPartitions);
+ } else {
+ return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+ }
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
index c165141dfc5..bd46569d827 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@@ -38,15 +40,30 @@ public class LogFileSizeBasedCompactionStrategy extends BoundedIOCompactionStrat
implements Comparator<HoodieCompactionOperation> {
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig writeConfig,
- List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+ List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionPlans) {
// Filter the file group which log files size is greater than the threshold in bytes.
// Order the operations based on the reverse size of the logs and limit them by the IO
long threshold = writeConfig.getCompactionLogFileSizeThreshold();
- return super.orderAndFilter(writeConfig, operations.stream()
- .filter(e -> e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) >= threshold)
- .sorted(this).collect(Collectors.toList()),
- pendingCompactionPlans);
+ ArrayList<String> missingPartitions = new ArrayList<>();
+ boolean incrementalTableServiceEnabled = writeConfig.isIncrementalTableServiceEnabled();
+ List<HoodieCompactionOperation> filterOperator = operations.stream()
+ .filter(e -> {
+ if (incrementalTableServiceEnabled && e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) < threshold) {
+ missingPartitions.add(e.getPartitionPath());
+ }
+ return e.getMetrics().getOrDefault(TOTAL_LOG_FILE_SIZE, 0d) >= threshold;
+ }).sorted(this).collect(Collectors.toList());
+
+ if (incrementalTableServiceEnabled) {
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+ List<HoodieCompactionOperation> compactOperations = resPair.getLeft();
+ List<String> innerMissingPartitions = resPair.getRight();
+ missingPartitions.addAll(innerMissingPartitions);
+ return Pair.of(compactOperations, missingPartitions);
+ } else {
+ return super.orderAndFilter(writeConfig, filterOperator, pendingCompactionPlans);
+ }
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
index eb3b42ee7ca..a9f49fd1fc4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
@@ -18,18 +18,23 @@
package org.apache.hudi.table.action.compact.strategy;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class PartitionRegexBasedCompactionStrategy extends CompactionStrategy {
+ /**
+ * Regex filtered partitions are not included in missing partitions.
+ */
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
String regex = writeConfig.getCompactionSpecifyPartitionPathRegex();
Pattern pattern = Pattern.compile(regex);
- return allPartitionPaths.stream().filter(pattern.asPredicate()).collect(Collectors.toList());
+ return Pair.of(allPartitionPaths.stream().filter(pattern.asPredicate()).collect(Collectors.toList()), Collections.emptyList());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
index ffc437bcd4c..d87354e42a3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedCompactionStrategy.java
@@ -20,8 +20,10 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import java.util.Collections;
import java.util.List;
/**
@@ -33,8 +35,9 @@ import java.util.List;
public class UnBoundedCompactionStrategy extends CompactionStrategy {
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig config,
- List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> pendingCompactionWorkloads) {
- return operations;
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig config,
+ List<HoodieCompactionOperation> operations,
+ List<HoodieCompactionPlan> pendingCompactionWorkloads) {
+ return Pair.of(operations, Collections.emptyList());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
index 66b5612e1ac..56ef8306c17 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
@@ -20,9 +20,11 @@ package org.apache.hudi.table.action.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@@ -39,27 +41,27 @@ import java.util.stream.Collectors;
public class UnBoundedPartitionAwareCompactionStrategy extends CompactionStrategy {
@Override
- public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig config,
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig config,
final List<HoodieCompactionOperation> operations, final List<HoodieCompactionPlan> pendingCompactionWorkloads) {
BoundedPartitionAwareCompactionStrategy boundedPartitionAwareCompactionStrategy =
new BoundedPartitionAwareCompactionStrategy();
List<HoodieCompactionOperation> operationsToExclude =
- boundedPartitionAwareCompactionStrategy.orderAndFilter(config, operations, pendingCompactionWorkloads);
+ boundedPartitionAwareCompactionStrategy.orderAndFilter(config, operations, pendingCompactionWorkloads).getLeft();
List<HoodieCompactionOperation> allOperations = new ArrayList<>(operations);
allOperations.removeAll(operationsToExclude);
- return allOperations;
+ return Pair.of(allOperations, Collections.emptyList());
}
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitionPaths) {
List<String> allPartitionPaths =
partitionPaths.stream().map(partition -> partition.replace("/", "-")).sorted(Comparator.reverseOrder())
.map(partitionPath -> partitionPath.replace("-", "/")).collect(Collectors.toList());
BoundedPartitionAwareCompactionStrategy boundedPartitionAwareCompactionStrategy =
new BoundedPartitionAwareCompactionStrategy();
List<String> partitionsToExclude =
- boundedPartitionAwareCompactionStrategy.filterPartitionPaths(writeConfig, partitionPaths);
+ boundedPartitionAwareCompactionStrategy.filterPartitionPaths(writeConfig, partitionPaths).getLeft();
allPartitionPaths.removeAll(partitionsToExclude);
- return allPartitionPaths;
+ return Pair.of(allPartitionPaths, Collections.emptyList());
}
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java
new file mode 100644
index 00000000000..84e21e821b4
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseTableServicePlanActionExecutor.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table;
+
+import org.apache.hudi.HoodieTestCommitGenerator;
+import org.apache.hudi.avro.model.HoodieClusteringGroup;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieClusteringStrategy;
+import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
+import org.apache.hudi.avro.model.HoodieSliceInfo;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.engine.TaskContextSupplier;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.testutils.FileCreateUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.table.action.BaseTableServicePlanActionExecutor;
+import org.apache.hudi.table.action.IncrementalPartitionAwareStrategy;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestBaseTableServicePlanActionExecutor {
+
+ @TempDir
+ File tempFile;
+ private static final StorageConfiguration<?> CONF = getDefaultStorageConf();
+ private final HoodieEngineContext context = new HoodieLocalEngineContext(CONF);
+
+ /**
+ * Clustering commit : cl
+ * Commit : c
+ * |requestTime<---commit type--->completionTime|(written partitions in current commit)
+ * ---------------------------------------------------------------------------------------------------------------------------------------------> timeline
+ * |0<-c1->1|(part=0000)
+ * |2<------cl2------>4|(part=0000)
+ * |3<-----------c3--------->6|(part=0003)
+ * |5<-----------c4--------->7|(part=0005)
+ * |8<-------------------c5----------------->N|(part=0008)
+ * | start incr clustering at 9
+ */
+ @Test
+ public void testExecutorWithMultiWriter() throws Exception {
+ String tableName = "testTable";
+ String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+ HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+ HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .withMarkersType("DIRECT")
+ .build();
+
+ // <requestTime, CompletionTime>
+ HashMap<String, String> instants = new HashMap<>();
+ instants.put("0000", "0001"); // commit 1
+ instants.put("0002", "0004"); // clustering 1
+ instants.put("0003", "0006"); // commit 2
+ instants.put("0005", "0007"); // commit 3
+
+ prepareTimeline(metaClient, instants, Collections.singletonList("0008"));
+
+ DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+ Set<String> incrementalPartitions = (Set<String>)executor.getIncrementalPartitions(TableServiceType.CLUSTER).getRight();
+
+ assertEquals(2, incrementalPartitions.size());
+ assertTrue(incrementalPartitions.contains("0003"));
+ assertTrue(incrementalPartitions.contains("0005"));
+ assertFalse(incrementalPartitions.contains("0008"));
+ }
+
+ @Test
+ public void testGetPartitionsFallbackToFullScan() throws Exception {
+ String tableName = "testTable";
+ String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+ HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+ HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .withMarkersType("DIRECT")
+ .build();
+
+ // <requestTime, CompletionTime>
+ HashMap<String, String> instants = new HashMap<>();
+ instants.put("0000", "0001");
+ instants.put("0002", "0004");
+ instants.put("0003", "0006");
+ instants.put("0005", "0007");
+ instants.put("0008", "0010");
+
+ prepareTimeline(metaClient, instants, Collections.emptyList());
+ instants.keySet().forEach(instant -> {
+ try {
+ FileCreateUtils.createPartitionMetaFile(tablePath, instant);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ DummyStrategy incrementalStrategy = new DummyStrategy();
+ DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+ List<String> partitions = (List<String>) executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+ assertEquals(2, partitions.size());
+
+ // Simulation archive commit instant 0000 and clustering instant 0002
+ Path path = new Path(tablePath);
+ FileSystem fs = path.getFileSystem(new Configuration());
+ StoragePath timelinePath = metaClient.getTimelinePath();
+ Arrays.stream(fs.listStatus(new Path(timelinePath.toString()))).forEach(instant -> {
+ if (instant.getPath().toString().contains("0000") || instant.getPath().toString().contains("0002")) {
+ try {
+ fs.delete(instant.getPath(), true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ // fall back to get all partitions
+ List<String> allPartitions = (List<String>) executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+ assertEquals(allPartitions.stream().sorted().collect(Collectors.toList()), instants.keySet().stream().sorted().collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testContinuousEmptyCommits() throws Exception {
+ String tableName = "testTable";
+ String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+ HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+ HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .withMarkersType("DIRECT")
+ .build();
+ HoodieTestTable testTable = HoodieTestTable.of(metaClient);
+
+ // create clustering commit 0001_0002
+ String clusteringRequestTime = "0001";
+ String clusteringCompletionTIme = "0002";
+ Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> replaceMetadata =
+ generateReplaceCommitMetadata(clusteringRequestTime, clusteringRequestTime, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+ testTable.addCluster(clusteringRequestTime, replaceMetadata.getKey(), Option.empty(), replaceMetadata.getValue(), clusteringCompletionTIme);
+
+ HoodieInstant clusteringInstant = metaClient.getActiveTimeline().filterCompletedInstants().getLastClusteringInstant().get();
+ FileCreateUtils.createPartitionMetaFile(tablePath, clusteringInstant.requestedTime());
+
+ // create empty commit 0003_004
+ String emptyCommitRequestTime = "0003";
+ String emptyCommitCompletionTIme = "0004";
+ HoodieCommitMetadata metadata = testTable.createCommitMetadata(emptyCommitRequestTime, WriteOperationType.INSERT, Collections.emptyList(), 0, false);
+ testTable.addCommit(emptyCommitRequestTime, Option.of(emptyCommitCompletionTIme), Option.of(metadata));
+
+ // create empty commit 0005_0006
+ String emptyCommitRequestTime2 = "0005";
+ String emptyCommitCompletionTIme2 = "0006";
+ HoodieCommitMetadata metadata2 = testTable.createCommitMetadata(emptyCommitRequestTime2, WriteOperationType.INSERT, Collections.emptyList(), 0, false);
+ testTable.addCommit(emptyCommitRequestTime2, Option.of(emptyCommitCompletionTIme2), Option.of(metadata2));
+
+ // executor.getPartitions should return empty list
+ DummyStrategy incrementalStrategy = new DummyStrategy();
+ DummyTableServicePlanActionExecutor executor = new DummyTableServicePlanActionExecutor(context, writeConfig, getMockHoodieTable(metaClient), "0009");
+ List partitions = executor.getPartitions(incrementalStrategy, TableServiceType.CLUSTER);
+ assertTrue(partitions.isEmpty());
+ }
+
+ private void prepareTimeline(HoodieTableMetaClient metaClient, HashMap<String, String> commitInstants,
+ List<String> requestInstants) throws Exception {
+ HoodieTestTable testTable = HoodieTestTable.of(metaClient);
+ commitInstants.forEach((requestTime, completionTime) -> {
+ try {
+ if (requestTime.equalsIgnoreCase("0002")) {
+ Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> replaceMetadata =
+ generateReplaceCommitMetadata(requestTime, requestTime, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+ testTable.addCluster(requestTime, replaceMetadata.getKey(), Option.empty(), replaceMetadata.getValue(), completionTime);
+ } else if (!requestTime.equalsIgnoreCase("0004")) {
+ HoodieCommitMetadata metadata = testTable.createCommitMetadata(requestTime, WriteOperationType.INSERT, Collections.singletonList(requestTime), 10, false);
+ testTable.addCommit(requestTime, Option.of(completionTime), Option.of(metadata));
+ }
+ } catch (Exception e) {
+ throw new HoodieException(e);
+ }
+ });
+
+ requestInstants.forEach(instant -> {
+ try {
+ testTable.addRequestedCommit(instant);
+ FileCreateUtils.createPartitionMetaFile(metaClient.getBasePath().toString(), instant);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private HoodieTable getMockHoodieTable(HoodieTableMetaClient metaClient) {
+ HoodieTable hoodieTable = mock(HoodieTable.class);
+ TaskContextSupplier taskContextSupplier = mock(TaskContextSupplier.class);
+ when(taskContextSupplier.getPartitionIdSupplier()).thenReturn(() -> 1);
+ when(hoodieTable.getTaskContextSupplier()).thenReturn(taskContextSupplier);
+ when(hoodieTable.getMetaClient()).thenReturn(metaClient);
+ when(hoodieTable.getActiveTimeline()).thenReturn(metaClient.getActiveTimeline());
+ return hoodieTable;
+ }
+
+ private Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> generateReplaceCommitMetadata(
+ String instantTime, String partition, String replacedFileId, String newFileId) {
+ HoodieRequestedReplaceMetadata requestedReplaceMetadata = new HoodieRequestedReplaceMetadata();
+ requestedReplaceMetadata.setOperationType(WriteOperationType.CLUSTER.toString());
+ requestedReplaceMetadata.setVersion(1);
+ HoodieSliceInfo sliceInfo = HoodieSliceInfo.newBuilder().setFileId(replacedFileId).build();
+ List<HoodieClusteringGroup> clusteringGroups = new ArrayList<>();
+ clusteringGroups.add(HoodieClusteringGroup.newBuilder()
+ .setVersion(1).setNumOutputFileGroups(1).setMetrics(Collections.emptyMap())
+ .setSlices(Collections.singletonList(sliceInfo)).build());
+ requestedReplaceMetadata.setExtraMetadata(Collections.emptyMap());
+ requestedReplaceMetadata.setClusteringPlan(HoodieClusteringPlan.newBuilder()
+ .setVersion(1).setExtraMetadata(Collections.emptyMap())
+ .setStrategy(HoodieClusteringStrategy.newBuilder().setStrategyClassName("").setVersion(1).build())
+ .setInputGroups(clusteringGroups).build());
+
+ HoodieReplaceCommitMetadata replaceMetadata = new HoodieReplaceCommitMetadata();
+ replaceMetadata.addReplaceFileId(partition, replacedFileId);
+ replaceMetadata.setOperationType(WriteOperationType.CLUSTER);
+ if (!StringUtils.isNullOrEmpty(newFileId)) {
+ HoodieWriteStat writeStat = new HoodieWriteStat();
+ writeStat.setPartitionPath(partition);
+ writeStat.setPath(partition + "/" + HoodieTestCommitGenerator.getBaseFilename(instantTime, newFileId));
+ writeStat.setFileId(newFileId);
+ writeStat.setTotalWriteBytes(1);
+ writeStat.setFileSizeInBytes(1);
+ replaceMetadata.addWriteStat(partition, writeStat);
+ }
+ return Pair.of(requestedReplaceMetadata, replaceMetadata);
+ }
+
+ class DummyTableServicePlanActionExecutor<T, I, K, O, R> extends BaseTableServicePlanActionExecutor<T, I, K, O, R> {
+
+ public DummyTableServicePlanActionExecutor(HoodieEngineContext context,
+ HoodieWriteConfig config,
+ HoodieTable<T, I, K, O> table,
+ String instantTime) {
+ super(context, config, table, instantTime);
+ }
+
+ @Override
+ public R execute() {
+ return null;
+ }
+ }
+
+ class DummyStrategy implements IncrementalPartitionAwareStrategy {
+
+ @Override
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> partitions) {
+ return null;
+ }
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
index a053a961105..e5ffda9fc53 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
@@ -18,7 +18,6 @@
package org.apache.hudi.table.action.cluster.strategy;
-import org.apache.hudi.avro.model.HoodieClusteringGroup;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -32,7 +31,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -83,11 +81,6 @@ public class TestPartitionAwareClusteringPlanStrategy {
super(table, engineContext, writeConfig);
}
- @Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List list) {
- return null;
- }
-
@Override
protected Map<String, String> getStrategyParams() {
return null;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
index a232bf54c16..a6684339b77 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategy.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -56,7 +57,7 @@ public class FlinkSizeBasedClusteringPlanStrategy<T>
}
@Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
index 10046f84f5b..4edf816dd47 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
@@ -34,6 +34,8 @@ import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor;
+import org.apache.hudi.util.Lazy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +63,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
}
@Override
- public Option<HoodieClusteringPlan> generateClusteringPlan() {
+ public Option<HoodieClusteringPlan> generateClusteringPlan(ClusteringPlanActionExecutor executor, Lazy<List<String>> partitions) {
if (!checkPrecondition()) {
return Option.empty();
}
@@ -71,7 +73,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
List<String> partitionPaths = getPartitionPathInActiveTimeline(hoodieTable);
- partitionPaths = filterPartitionPaths(partitionPaths);
+ partitionPaths = filterPartitionPaths(getWriteConfig(), partitionPaths).getLeft();
if (partitionPaths.isEmpty()) {
// In case no partitions could be picked, return no clustering plan
@@ -82,7 +84,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa
.flatMap(
partitionPaths, partitionPath -> {
List<FileSlice> fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
- return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).limit(getWriteConfig().getClusteringMaxNumGroups());
+ return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).getLeft().limit(getWriteConfig().getClusteringMaxNumGroups());
},
partitionPaths.size())
.stream()
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
index 36cb5ebdf90..e31570f7d02 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
@@ -74,7 +74,7 @@ public class TestFlinkSizeBasedClusteringPlanStrategy {
.build())
.build();
PartitionAwareClusteringPlanStrategy strategyWithSortEnabled = new FlinkSizeBasedClusteringPlanStrategy(table, context, configWithSortEnabled);
- Stream<HoodieClusteringGroup> groupStreamSort = strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+ Stream<HoodieClusteringGroup> groupStreamSort = (Stream<HoodieClusteringGroup>) strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
assertEquals(1, groupStreamSort.count());
// test buildClusteringGroupsForPartition without ClusteringSortColumns config
@@ -86,7 +86,7 @@ public class TestFlinkSizeBasedClusteringPlanStrategy {
.build())
.build();
PartitionAwareClusteringPlanStrategy strategyWithSortDisabled = new FlinkSizeBasedClusteringPlanStrategy(table, context, configWithSortDisabled);
- Stream<HoodieClusteringGroup> groupStreamWithOutSort = strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+ Stream<HoodieClusteringGroup> groupStreamWithOutSort = (Stream<HoodieClusteringGroup>) strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
assertEquals(0, groupStreamWithOutSort.count());
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
index 03d416452d2..e0309e50e75 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/JavaSizeBasedClusteringPlanStrategy.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -57,7 +58,7 @@ public class JavaSizeBasedClusteringPlanStrategy<T>
}
@Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
index ba3f917d04c..9e6a76e398a 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSingleFileSortPlanStrategy.java
@@ -43,13 +43,13 @@ public class SparkSingleFileSortPlanStrategy<T>
}
@Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
List<Pair<List<FileSlice>, Integer>> fileSliceGroups = fileSlices.stream()
.map(fileSlice -> Pair.of(Collections.singletonList(fileSlice), 1)).collect(Collectors.toList());
- return fileSliceGroups.stream().map(fileSliceGroup -> HoodieClusteringGroup.newBuilder()
+ return Pair.of(fileSliceGroups.stream().map(fileSliceGroup -> HoodieClusteringGroup.newBuilder()
.setSlices(getFileSliceInfo(fileSliceGroup.getLeft()))
.setNumOutputFileGroups(fileSliceGroup.getRight())
.setMetrics(buildMetrics(fileSliceGroup.getLeft()))
- .build());
+ .build()), true);
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
index 2c82810d175..0d3bac365bd 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy;
@@ -57,7 +58,7 @@ public class SparkSizeBasedClusteringPlanStrategy<T>
}
@Override
- protected Stream<HoodieClusteringGroup> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
+ protected Pair<Stream<HoodieClusteringGroup>, Boolean> buildClusteringGroupsForPartition(String partitionPath, List<FileSlice> fileSlices) {
return super.buildClusteringGroupsForPartition(partitionPath, fileSlices);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
index ad9c5b45a0f..4d3d40b1799 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
@@ -67,7 +67,7 @@ public class TestSparkSizeBasedClusteringPlanStrategy {
fileSlices.add(createFileSlice(400));
fileSlices.add(createFileSlice(400));
- Stream<HoodieClusteringGroup> clusteringGroupStream = planStrategy.buildClusteringGroupsForPartition("p0", fileSlices);
+ Stream<HoodieClusteringGroup> clusteringGroupStream = (Stream<HoodieClusteringGroup>) planStrategy.buildClusteringGroupsForPartition("p0", fileSlices).getLeft();
List<HoodieClusteringGroup> clusteringGroups = clusteringGroupStream.collect(Collectors.toList());
// FileSlices will be divided into two clusteringGroups
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
index aed4cff3ef1..a878cf43817 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/ClusteringTestUtils.java
@@ -26,7 +26,10 @@ import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.CompactionTestUtils;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
@@ -111,6 +114,16 @@ public class ClusteringTestUtils {
return runClusteringOnInstant(clusteringClient, skipExecution, shouldCommit, clusteringCommitTime);
}
+ public static HoodieClusteringPlan getClusteringPlan(HoodieTableMetaClient client, HoodieInstant clusteringInstant) {
+ return ClusteringUtils.getClusteringPlan(client, clusteringInstant).get().getRight();
+ }
+
+ public static HoodieClusteringPlan getClusteringPlan(HoodieTableMetaClient client, String clusteringInstantString) {
+ HoodieInstant clusteringInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.CLUSTERING_ACTION,
+ clusteringInstantString, InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ return getClusteringPlan(client, clusteringInstant);
+ }
+
public static String runClusteringOnInstant(SparkRDDWriteClient clusteringClient, boolean skipExecution, boolean shouldCommit, String clusteringCommitTime) {
clusteringClient.scheduleClusteringAtInstant(clusteringCommitTime, Option.empty());
if (!skipExecution) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java
new file mode 100644
index 00000000000..970954e0637
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieSliceInfo;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.TableServiceType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.config.HoodieClusteringConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.table.HoodieSparkTable;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.cluster.strategy.ClusteringPlanStrategy;
+import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.config.HoodieClusteringConfig.DAYBASED_LOOKBACK_PARTITIONS;
+import static org.apache.hudi.config.HoodieClusteringConfig.PARTITION_FILTER_BEGIN_PARTITION;
+import static org.apache.hudi.config.HoodieClusteringConfig.PARTITION_FILTER_END_PARTITION;
+import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_PARTITION_FILTER_MODE;
+import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestIncrementalClustering extends SparkClientFunctionalTestHarness {
+
+ private static final String TODAY;
+ private static final String YESTERDAY;
+ private static final String TOMORROW;
+
+ static {
+ LocalDate today = LocalDate.now();
+ LocalDate yesterday = today.minusDays(1);
+ LocalDate tomorrow = today.plusDays(1);
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
+
+ TODAY = today.format(formatter);
+ YESTERDAY = yesterday.format(formatter);
+ TOMORROW = tomorrow.format(formatter);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testPartitionsForIncrClusteringWithMaxGroupLimit(boolean enableIncrTableService) throws Exception {
+ int maxClusteringGroup = 1;
+ HoodieWriteConfig writeConfig = buildWriteConfig(enableIncrTableService, new Properties(), maxClusteringGroup);
+ HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.COPY_ON_WRITE);
+ String[] partitions = {YESTERDAY, TODAY};
+ // write twice, make sure there are multi-slices in each partitions
+ prepareBasicData(writeConfig, partitions);
+ prepareBasicData(writeConfig, partitions);
+ String clusteringInstantTime = doClustering(writeConfig);
+
+ HoodieClusteringPlan clusteringPlan = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime);
+ if (enableIncrTableService) {
+ // Since setting maxClusteringGroup 1, and there are 2 slices under each partition,
+ // all files in each partition have not been completely processed, that is, they are all missing partitions.
+ assertEquals(2, clusteringPlan.getMissingSchedulePartitions().size());
+ assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(YESTERDAY));
+ assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(TODAY));
+
+ String[] partitions2 = {TOMORROW};
+ prepareBasicData(buildWriteConfig(true, new Properties(), maxClusteringGroup), partitions2);
+ HoodieSparkTable table = HoodieSparkTable.create(writeConfig, context());
+ ClusteringPlanActionExecutor executor =
+ new ClusteringPlanActionExecutor<>(context(), writeConfig, table, "9999999", Option.empty());
+
+ List<String> incrementalPartitions = executor.getPartitions(ReflectionUtils.loadClass(
+ ClusteringPlanStrategy.checkAndGetClusteringPlanStrategy(writeConfig),
+ new Class<?>[] {HoodieTable.class, HoodieEngineContext.class, HoodieWriteConfig.class}, table, context(), writeConfig), TableServiceType.CLUSTER);
+ // fetch 2 missing partitions from last completed clustering plan as incremental partitions
+ // get 1 partitions from new commit as incremental partitions
+ assertEquals(3, incrementalPartitions.size());
+ } else {
+ assertNull(clusteringPlan.getMissingSchedulePartitions());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("testIncrClusteringWithFilter")
+ public void testPartitionsForIncrClusteringWithFilter(ClusteringPlanPartitionFilterMode mode,
+ Properties props) throws Exception {
+ HoodieWriteConfig writeConfig = buildWriteConfig(true, props, 100);
+ HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.COPY_ON_WRITE);
+ String[] partitions = {YESTERDAY, TODAY};
+ // write twice, make sure there are multi-slices in each partitions
+ prepareBasicData(writeConfig, partitions);
+ String clusteringInstantTime = doClustering(writeConfig);
+ HoodieClusteringPlan clusteringPlan = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime);
+
+ switch (mode) {
+ case NONE: {
+ // For partitions filtered out by the regex expression, they will not be recorded in the missingPartitions
+ assertEquals(0, clusteringPlan.getMissingSchedulePartitions().size());
+ break;
+ }
+ case SELECTED_PARTITIONS: {
+ Set<String> affectedPartitions = getAffectedPartition(clusteringPlan);
+ assertEquals(1, affectedPartitions.size());
+ assertTrue(affectedPartitions.contains(YESTERDAY));
+ assertEquals(0, clusteringPlan.getMissingSchedulePartitions().size());
+ break;
+ }
+ case RECENT_DAYS: {
+ assertEquals(1, clusteringPlan.getMissingSchedulePartitions().size());
+ assertTrue(clusteringPlan.getMissingSchedulePartitions().contains(TODAY));
+ // do another TOMORROW ingestion and clustering
+ String[] partitions2 = {TOMORROW};
+ HoodieWriteConfig hoodieWriteConfig2 = buildWriteConfig(true, props, 100);
+ prepareBasicData(hoodieWriteConfig2, partitions2);
+ String clusteringInstantTime2 = doClustering(hoodieWriteConfig2);
+ HoodieClusteringPlan clusteringPlan2 = ClusteringTestUtils.getClusteringPlan(metaClient, clusteringInstantTime2);
+ assertEquals(1, clusteringPlan2.getMissingSchedulePartitions().size());
+ assertTrue(clusteringPlan2.getMissingSchedulePartitions().contains(TOMORROW));
+ break;
+ }
+ default: {
+ throw new HoodieException("Un-support mode" + mode);
+ }
+ }
+ }
+
+ private Set<String> getAffectedPartition(HoodieClusteringPlan clusteringPlan) {
+ return clusteringPlan.getInputGroups().stream().flatMap(hoodieClusteringGroup -> {
+ return hoodieClusteringGroup.getSlices().stream();
+ }).map(HoodieSliceInfo::getPartitionPath).collect(Collectors.toSet());
+ }
+
+ public static Stream<Object> testIncrClusteringWithFilter() {
+ Properties none = new Properties();
+ none.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.NONE);
+ none.put("hoodie.clustering.plan.strategy.partition.regex.pattern", TODAY + ".*");
+
+ Properties selectedPartitions = new Properties();
+ selectedPartitions.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.SELECTED_PARTITIONS.name());
+ selectedPartitions.put(PARTITION_FILTER_BEGIN_PARTITION.key(), YESTERDAY);
+ selectedPartitions.put(PARTITION_FILTER_END_PARTITION.key(), YESTERDAY);
+
+ Properties recentDay = new Properties();
+ recentDay.put(PLAN_PARTITION_FILTER_MODE, ClusteringPlanPartitionFilterMode.RECENT_DAYS);
+ recentDay.put(PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST.key(), 1);
+ recentDay.put(DAYBASED_LOOKBACK_PARTITIONS.key(), 1);
+
+ return Stream.of(
+ Arguments.of(ClusteringPlanPartitionFilterMode.NONE, none),
+ Arguments.of(ClusteringPlanPartitionFilterMode.SELECTED_PARTITIONS, selectedPartitions),
+ Arguments.of(ClusteringPlanPartitionFilterMode.RECENT_DAYS, recentDay)
+ );
+ }
+
+ private HoodieWriteConfig buildWriteConfig(boolean enableIncrTableService, Properties properties, int maxClusteringGroup) {
+ properties.put("hoodie.datasource.write.row.writer.enable", String.valueOf(false));
+ properties.put("hoodie.parquet.small.file.limit", String.valueOf(-1));
+ return getConfigBuilder(true)
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
+ .withClusteringConfig(HoodieClusteringConfig.newBuilder()
+ .withClusteringMaxNumGroups(maxClusteringGroup)
+ .withClusteringMaxBytesInGroup(maxClusteringGroup)
+ .fromProperties(properties)
+ .build())
+ .withProperties(properties)
+ .build();
+ }
+
+ private void prepareBasicData(HoodieWriteConfig writeConfig, String[] partitions) throws IOException {
+ SparkRDDWriteClient client = getHoodieWriteClient(writeConfig);
+ HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(partitions);
+ for (int i = 0; i < partitions.length; i++) {
+ String instantTime = client.createNewInstantTime();
+ client.startCommitWithTime(instantTime);
+ client.insert(jsc().parallelize(dataGen.generateInsertsForPartition(instantTime, 10, partitions[i]), 1), instantTime);
+ }
+ client.close();
+ }
+
+ private String doClustering(HoodieWriteConfig writeConfig) throws IOException {
+ try (SparkRDDWriteClient client = getHoodieWriteClient(writeConfig)) {
+ return ClusteringTestUtils.runClustering(client, false, true);
+ }
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
index ada5f4954ab..5e31d2ee4f3 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
@@ -73,7 +73,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
.build())
.build();
PartitionAwareClusteringPlanStrategy strategyWithSortEnabled = new SparkSizeBasedClusteringPlanStrategy(table, context, configWithSortEnabled);
- Stream<HoodieClusteringGroup> groupStreamSort = strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+ Stream<HoodieClusteringGroup> groupStreamSort = (Stream<HoodieClusteringGroup>) strategyWithSortEnabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
assertEquals(1, groupStreamSort.count());
// test buildClusteringGroupsForPartition without ClusteringSortColumns config
@@ -85,7 +85,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
.build())
.build();
PartitionAwareClusteringPlanStrategy strategyWithSortDisabled = new SparkSizeBasedClusteringPlanStrategy(table, context, configWithSortDisabled);
- Stream<HoodieClusteringGroup> groupStreamWithOutSort = strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+ Stream<HoodieClusteringGroup> groupStreamWithOutSort = (Stream<HoodieClusteringGroup>) strategyWithSortDisabled.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
assertEquals(0, groupStreamWithOutSort.count());
}
@@ -107,7 +107,7 @@ public class TestSparkBuildClusteringGroupsForPartition {
.build())
.build();
PartitionAwareClusteringPlanStrategy clusteringPlanStrategy = new SparkSizeBasedClusteringPlanStrategy(table, context, writeConfig);
- Stream<HoodieClusteringGroup> groups = clusteringPlanStrategy.buildClusteringGroupsForPartition(partition,fileSliceGroups);
+ Stream<HoodieClusteringGroup> groups = (Stream<HoodieClusteringGroup>) clusteringPlanStrategy.buildClusteringGroupsForPartition(partition,fileSliceGroups).getLeft();
assertEquals(2, groups.count());
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
index 70643a327d4..a173f4df80b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkClusteringPlanPartitionFilter.java
@@ -63,7 +63,7 @@ public class TestSparkClusteringPlanPartitionFilter {
fakeTimeBasedPartitionsPath.add("20210718");
fakeTimeBasedPartitionsPath.add("20210716");
fakeTimeBasedPartitionsPath.add("20210719");
- List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+ List list = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
assertEquals(3, list.size());
}
@@ -81,7 +81,7 @@ public class TestSparkClusteringPlanPartitionFilter {
fakeTimeBasedPartitionsPath.add("20210718");
fakeTimeBasedPartitionsPath.add("20210716");
fakeTimeBasedPartitionsPath.add("20210719");
- List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+ List list = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
assertEquals(1, list.size());
assertSame("20210718", list.get(0));
}
@@ -101,7 +101,7 @@ public class TestSparkClusteringPlanPartitionFilter {
fakeTimeBasedPartitionsPath.add("20211221");
fakeTimeBasedPartitionsPath.add("20211222");
fakeTimeBasedPartitionsPath.add("20211224");
- List list = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+ List list = (List)sg.filterPartitionPaths(table.getConfig(), fakeTimeBasedPartitionsPath).getLeft();
assertEquals(1, list.size());
assertSame("20211222", list.get(0));
}
@@ -117,7 +117,7 @@ public class TestSparkClusteringPlanPartitionFilter {
for (int i = 0; i < 24; i++) {
fakeTimeBasedPartitionsPath.add("20220301" + (i >= 10 ? String.valueOf(i) : "0" + i));
}
- List filterPartitions = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+ List filterPartitions = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
assertEquals(1, filterPartitions.size());
assertEquals(fakeTimeBasedPartitionsPath.get(DateTime.now().getHourOfDay()), filterPartitions.get(0));
fakeTimeBasedPartitionsPath = new ArrayList<>();
@@ -125,7 +125,7 @@ public class TestSparkClusteringPlanPartitionFilter {
fakeTimeBasedPartitionsPath.add("20220301" + (i >= 10 ? String.valueOf(i) : "0" + i));
fakeTimeBasedPartitionsPath.add("20220302" + (i >= 10 ? String.valueOf(i) : "0" + i));
}
- filterPartitions = sg.filterPartitionPaths(fakeTimeBasedPartitionsPath);
+ filterPartitions = (List)sg.filterPartitionPaths(null, fakeTimeBasedPartitionsPath).getLeft();
assertEquals(2, filterPartitions.size());
int hourOfDay = DateTime.now().getHourOfDay();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
index a368d43da25..8187b22bd9a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
@@ -30,6 +30,8 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -52,8 +54,9 @@ public class TestHoodieCompactionStrategy {
private static final Random RANDOM = new Random();
private String[] partitionPaths = {"2017/01/01", "2017/01/02", "2017/01/03"};
- @Test
- public void testUnBounded() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testUnBounded(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -61,26 +64,39 @@ public class TestHoodieCompactionStrategy {
sizesMap.put(90 * MB, Collections.singletonList(1024 * MB));
UnBoundedCompactionStrategy strategy = new UnBoundedCompactionStrategy();
HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).build()).build();
- List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap).getLeft();
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy()
+ .orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+
assertEquals(operations, returned, "UnBounded should not re-order or filter");
+ assertEquals(missingPartitions.size(), 0);
}
- @Test
- public void testBoundedIOSimple() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testBoundedIOSimple(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
sizesMap.put(100 * MB, Collections.singletonList(MB));
sizesMap.put(90 * MB, Collections.singletonList(1024 * MB));
BoundedIOCompactionStrategy strategy = new BoundedIOCompactionStrategy();
- HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
+ .withCompactionConfig(
HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(400).build())
.build();
- List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
-
+ List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap).getLeft();
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ assertEquals(1, missingPartitions.stream().distinct().count());
+ }
assertTrue(returned.size() < operations.size(), "BoundedIOCompaction should have resulted in fewer compactions");
assertEquals(2, returned.size(), "BoundedIOCompaction should have resulted in 2 compactions being chosen");
// Total size of all the log files
@@ -90,8 +106,9 @@ public class TestHoodieCompactionStrategy {
"Should chose the first 2 compactions which should result in a total IO of 610 MB");
}
- @Test
- public void testLogFileSizeCompactionSimple() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testLogFileSizeCompactionSimple(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -102,9 +119,17 @@ public class TestHoodieCompactionStrategy {
HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(1205)
.withLogFileSizeThresholdBasedCompaction(100 * 1024 * 1024).build())
.build();
- List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
-
+ Pair<List<HoodieCompactionOperation>, Map<Long, String>> operationAndPartition = createCompactionOperations(writeConfig, sizesMap);
+ List<HoodieCompactionOperation> operations = operationAndPartition.getLeft();
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ String missedPart1 = operationAndPartition.getRight().get(110 * MB);
+ String missedPart2 = operationAndPartition.getRight().get(100 * MB);
+ assertTrue(missingPartitions.contains(missedPart1));
+ assertTrue(missingPartitions.contains(missedPart2));
+ }
assertTrue(returned.size() < operations.size(),
"LogFileSizeBasedCompactionStrategy should have resulted in fewer compactions");
assertEquals(2, returned.size(), "LogFileSizeBasedCompactionStrategy should have resulted in 2 compaction");
@@ -115,8 +140,9 @@ public class TestHoodieCompactionStrategy {
"Should chose the first 2 compactions which should result in a total IO of 1594 MB");
}
- @Test
- public void testDayBasedCompactionSimple() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDayBasedCompactionSimple(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -134,10 +160,16 @@ public class TestHoodieCompactionStrategy {
DayBasedCompactionStrategy strategy = new DayBasedCompactionStrategy();
HoodieWriteConfig writeConfig =
- HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ HoodieWriteConfig.newBuilder().withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(1).build()).build();
-
- List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+ Pair<List<String>, List<String>> resPair = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+ List<String> filterPartitions = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ assertTrue(missingPartitions.isEmpty());
+ }
assertEquals(1, filterPartitions.size(), "DayBasedCompactionStrategy should have resulted in fewer partitions");
List<HoodieCompactionOperation> operations = createCompactionOperationsForPartition(writeConfig, sizesMap, keyToPartitionMap, filterPartitions);
@@ -155,8 +187,9 @@ public class TestHoodieCompactionStrategy {
assertTrue(comparison >= 0, "DayBasedCompactionStrategy should sort partitions in descending order");
}
- @Test
- public void testDayBasedCompactionWithIOBounded() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDayBasedCompactionWithIOBounded(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -175,6 +208,7 @@ public class TestHoodieCompactionStrategy {
DayBasedCompactionStrategy strategy = new DayBasedCompactionStrategy();
HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
.withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCompactionStrategy(strategy)
.withTargetPartitionsPerDayBasedCompaction(1)
@@ -182,11 +216,17 @@ public class TestHoodieCompactionStrategy {
.build())
.build();
- List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths));
+ List<String> filterPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, Arrays.asList(partitionPaths)).getLeft();
assertEquals(1, filterPartitions.size(), "DayBasedCompactionStrategy should have resulted in fewer partitions");
List<HoodieCompactionOperation> operations = createCompactionOperationsForPartition(writeConfig, sizesMap, keyToPartitionMap, filterPartitions);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ assertEquals(1, missingPartitions.size());
+ assertTrue(missingPartitions.contains(partitionPaths[2]));
+ }
assertEquals(1, returned.size(),
"DayBasedAndBoundedIOCompactionStrategy should have resulted in fewer compactions");
@@ -204,9 +244,10 @@ public class TestHoodieCompactionStrategy {
assertEquals(390, (long) returnedSize,
"Should chose the first and the third compactions which should result in a total IO of 591 MB");
}
-
- @Test
- public void testBoundedPartitionAwareCompactionSimple() {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testBoundedPartitionAwareCompactionSimple(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -238,10 +279,17 @@ public class TestHoodieCompactionStrategy {
BoundedPartitionAwareCompactionStrategy strategy = new BoundedPartitionAwareCompactionStrategy();
HoodieWriteConfig writeConfig =
- HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ HoodieWriteConfig.newBuilder().withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(2).build()).build();
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ assertTrue(missingPartitions.isEmpty());
+ }
assertTrue(returned.size() < operations.size(),
"BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions");
@@ -254,8 +302,9 @@ public class TestHoodieCompactionStrategy {
assertTrue(comparison >= 0, "BoundedPartitionAwareCompactionStrategy should sort partitions in descending order");
}
- @Test
- public void testUnboundedPartitionAwareCompactionSimple() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testUnboundedPartitionAwareCompactionSimple(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -287,10 +336,17 @@ public class TestHoodieCompactionStrategy {
UnBoundedPartitionAwareCompactionStrategy strategy = new UnBoundedPartitionAwareCompactionStrategy();
HoodieWriteConfig writeConfig =
- HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ HoodieWriteConfig.newBuilder().withPath("/tmp")
+ .withIncrementalTableServiceEnabled(enableIncrTableService)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCompactionStrategy(strategy).withTargetPartitionsPerDayBasedCompaction(2).build()).build();
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ assertTrue(missingPartitions.isEmpty());
+ }
assertTrue(returned.size() < operations.size(),
"UnBoundedPartitionAwareCompactionStrategy should not include last "
@@ -299,8 +355,9 @@ public class TestHoodieCompactionStrategy {
"BoundedPartitionAwareCompactionStrategy should have resulted in 1 compaction");
}
- @Test
- public void testLogFileLengthBasedCompactionStrategy() {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testLogFileLengthBasedCompactionStrategy(boolean enableIncrTableService) {
Map<Long, List<Long>> sizesMap = new HashMap<>();
sizesMap.put(120 * MB, Arrays.asList(60 * MB, 10 * MB, 80 * MB));
sizesMap.put(110 * MB, new ArrayList<>());
@@ -311,8 +368,16 @@ public class TestHoodieCompactionStrategy {
HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).withTargetIOPerCompactionInMB(1024)
.withCompactionLogFileNumThreshold(2).build())
.build();
- List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
- List<HoodieCompactionOperation> returned = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ Pair<List<HoodieCompactionOperation>, Map<Long, String>> operationAndPartition = createCompactionOperations(writeConfig, sizesMap);
+ List<HoodieCompactionOperation> operations = operationAndPartition.getLeft();
+ Pair<List<HoodieCompactionOperation>, List<String>> resPair = writeConfig.getCompactionStrategy().orderAndFilter(writeConfig, operations, new ArrayList<>());
+ List<HoodieCompactionOperation> returned = resPair.getLeft();
+ List<String> missingPartitions = resPair.getRight();
+ if (enableIncrTableService) {
+ Map<Long, String> partition = operationAndPartition.getRight();
+ assertTrue(missingPartitions.contains(partition.get(110 * MB)));
+ assertTrue(missingPartitions.contains(partition.get(100 * MB)));
+ }
assertTrue(returned.size() < operations.size(),
"LogFileLengthBasedCompactionStrategy should have resulted in fewer compactions");
@@ -341,7 +406,7 @@ public class TestHoodieCompactionStrategy {
List<String> allPartitionPaths = Arrays.asList(
"2017/01/01", "2018/01/02", "2017/02/01"
);
- List<String> returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths);
+ List<String> returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths).getLeft();
// filter by num first and then filter by prefix
assertEquals(1, returned.size());
assertEquals("2017/01/01", returned.get(0));
@@ -349,7 +414,7 @@ public class TestHoodieCompactionStrategy {
writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
HoodieCompactionConfig.newBuilder().withCompactionStrategy(new PrefixStrategy(), new NumStrategy()).withTargetIOPerCompactionInMB(1024)
.withCompactionLogFileNumThreshold(2).build()).build();
- returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths);
+ returned = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, allPartitionPaths).getLeft();
// filter by prefix first and then filter by num
assertEquals(2, returned.size());
assertEquals("2017/01/01", returned.get(0));
@@ -358,15 +423,28 @@ public class TestHoodieCompactionStrategy {
public static class NumStrategy extends CompactionStrategy {
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
- return allPartitionPaths.stream().limit(2).collect(Collectors.toList());
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ List<String> partitionToProcess = allPartitionPaths.stream().limit(2).collect(Collectors.toList());
+ List<String> missingPartitions = allPartitionPaths.stream().skip(2).collect(Collectors.toList());
+ return Pair.of(partitionToProcess, missingPartitions);
}
}
public static class PrefixStrategy extends CompactionStrategy {
@Override
- public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
- return allPartitionPaths.stream().filter(s -> s.startsWith("2017")).collect(Collectors.toList());
+ public Pair<List<String>, List<String>> filterPartitionPaths(HoodieWriteConfig writeConfig, List<String> allPartitionPaths) {
+ ArrayList<String> partitionsToProcess = new ArrayList<>();
+ ArrayList<String> missingPartitions = new ArrayList<>();
+
+ allPartitionPaths.forEach(partition -> {
+ if (partition.startsWith("2017")) {
+ partitionsToProcess.add(partition);
+ } else {
+ missingPartitions.add(partition);
+ }
+ });
+
+ return Pair.of(partitionsToProcess, missingPartitions);
}
}
@@ -381,21 +459,21 @@ public class TestHoodieCompactionStrategy {
);
HoodieWriteConfig writeConfig = updateRegex(".*");
- List<String> filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ List<String> filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(5, filteredPartitions.size());
writeConfig = updateRegex("2020/01/01");
- filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(1, filteredPartitions.size());
assertEquals("2020/01/01", filteredPartitions.get(0));
writeConfig = updateRegex("2020/01/0[1-2]");
- filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(2, filteredPartitions.size());
assertEquals("2020/01/01", filteredPartitions.get(0));
assertEquals("2020/01/02", filteredPartitions.get(1));
writeConfig = updateRegex("2020/01/0[1-2]|2020/02/01");
- filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(3, filteredPartitions.size());
assertEquals("2020/01/01", filteredPartitions.get(0));
assertEquals("2020/01/02", filteredPartitions.get(1));
@@ -403,13 +481,13 @@ public class TestHoodieCompactionStrategy {
writeConfig = updateRegex("2020/.*/01");
- filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(2, filteredPartitions.size());
assertEquals("2020/01/01", filteredPartitions.get(0));
assertEquals("2020/02/01", filteredPartitions.get(1));
writeConfig = updateRegex(".*/01/.*");
- filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions);
+ filteredPartitions = writeConfig.getCompactionStrategy().filterPartitionPaths(writeConfig, partitions).getLeft();
assertEquals(4, filteredPartitions.size());
assertEquals("2020/01/01", filteredPartitions.get(0));
assertEquals("2020/01/02", filteredPartitions.get(1));
@@ -426,12 +504,12 @@ public class TestHoodieCompactionStrategy {
return writeConfig;
}
- private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,
+ private Pair<List<HoodieCompactionOperation>, Map<Long, String>> createCompactionOperations(HoodieWriteConfig config,
Map<Long, List<Long>> sizesMap) {
Map<Long, String> keyToPartitionMap = sizesMap.keySet().stream()
.map(e -> Pair.of(e, partitionPaths[RANDOM.nextInt(partitionPaths.length - 1)]))
.collect(Collectors.toMap(Pair::getKey, Pair::getValue));
- return createCompactionOperations(config, sizesMap, keyToPartitionMap);
+ return Pair.of(createCompactionOperations(config, sizesMap, keyToPartitionMap), keyToPartitionMap);
}
private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,
diff --git a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
index 87486267d1c..d4a6c207762 100644
--- a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
+++ b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
@@ -50,6 +50,14 @@
"name":"preserveHoodieMetadata",
"type":["null", "boolean"],
"default": null
- }
+ },
+ {
+ "name":"missingSchedulePartitions",
+ "type":["null", {
+ "type":"array",
+ "items":"string"
+ }],
+ "default": null
+ }
]
}
diff --git a/hudi-common/src/main/avro/HoodieCompactionOperation.avsc b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
index bab7321f29c..61a96faec87 100644
--- a/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
+++ b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc
@@ -96,6 +96,14 @@
"name":"preserveHoodieMetadata",
"type":["boolean", "null"],
"default": false
+ },
+ {
+ "name":"missingSchedulePartitions",
+ "type":["null", {
+ "type":"array",
+ "items":"string"
+ }],
+ "default": null
}
]
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
index 2bc8f7fba62..eaed3668c9f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java
@@ -192,7 +192,7 @@ public class CompactionTestUtils {
}
}).collect(Collectors.toList());
return new HoodieCompactionPlan(ops.isEmpty() ? null : ops, new HashMap<>(),
- CompactionUtils.LATEST_COMPACTION_METADATA_VERSION, null, null);
+ CompactionUtils.LATEST_COMPACTION_METADATA_VERSION, null, null, null);
}
/**
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
index f2bafc575ee..ac1a1a3dd10 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -305,6 +305,11 @@ public class FileCreateUtils {
serializeCommitMetadata(commitMetadataSerDe, metadata).get());
}
+ public static void createReplaceCommit(CommitMetadataSerDe commitMetadataSerDe, String basePath,
+ String instantTime, String completionTime, HoodieReplaceCommitMetadata metadata) throws IOException {
+ createMetaFile(basePath, instantTime, () -> completionTime, HoodieTimeline.REPLACE_COMMIT_EXTENSION, serializeCommitMetadata(commitMetadataSerDe, metadata).get());
+ }
+
public static void createRequestedClusterCommit(String basePath, String instantTime,
HoodieRequestedReplaceMetadata requestedReplaceMetadata)
throws IOException {
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
index f35a2792731..c40dd2350e5 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
@@ -131,7 +131,7 @@ public class TestClusteringUtil {
private String generateClusteringPlan() {
HoodieClusteringGroup group = new HoodieClusteringGroup();
HoodieClusteringPlan plan = new HoodieClusteringPlan(Collections.singletonList(group),
- HoodieClusteringStrategy.newBuilder().build(), Collections.emptyMap(), 1, false);
+ HoodieClusteringStrategy.newBuilder().build(), Collections.emptyMap(), 1, false, null);
HoodieRequestedReplaceMetadata metadata = new HoodieRequestedReplaceMetadata(WriteOperationType.CLUSTER.name(),
plan, Collections.emptyMap(), 1);
String instantTime = table.getMetaClient().createNewInstantTime();
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
index 111680bdcdb..ccfd5965cad 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestCompactionUtil.java
@@ -174,7 +174,7 @@ public class TestCompactionUtil {
*/
private String generateCompactionPlan() {
HoodieCompactionOperation operation = new HoodieCompactionOperation();
- HoodieCompactionPlan plan = new HoodieCompactionPlan(Collections.singletonList(operation), Collections.emptyMap(), 1, null, null);
+ HoodieCompactionPlan plan = new HoodieCompactionPlan(Collections.singletonList(operation), Collections.emptyMap(), 1, null, null, null);
String instantTime = table.getMetaClient().createNewInstantTime();
HoodieInstant compactionInstant =
INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instantTime);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index c3e221b28e7..5353898136f 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -376,6 +376,19 @@ public class HoodieTestTable implements AutoCloseable {
return this;
}
+ public HoodieTestTable addCluster(
+ String instantTime,
+ HoodieRequestedReplaceMetadata requestedReplaceMetadata,
+ Option<HoodieCommitMetadata> inflightReplaceMetadata,
+ HoodieReplaceCommitMetadata completeReplaceMetadata,
+ String completionTime) throws Exception {
+ createRequestedClusterCommit(basePath, instantTime, requestedReplaceMetadata);
+ createInflightClusterCommit(COMMIT_METADATA_SER_DE, basePath, instantTime, inflightReplaceMetadata);
+ createReplaceCommit(COMMIT_METADATA_SER_DE, basePath, instantTime, completionTime, completeReplaceMetadata);
+ currentInstantTime = instantTime;
+ return this;
+ }
+
public HoodieTestTable addRequestedReplace(String instantTime, Option<HoodieRequestedReplaceMetadata> requestedReplaceMetadata) throws Exception {
createRequestedReplaceCommit(basePath, instantTime, requestedReplaceMetadata);
currentInstantTime = instantTime;
@@ -1324,6 +1337,10 @@ public class HoodieTestTable implements AutoCloseable {
HoodieTableType tableType,
String commitTime,
Map<String, List<Pair<String, Integer>>> partitionToFilesNameLengthMap) {
+ if (partitionToFilesNameLengthMap.isEmpty()) {
+ return testTableState;
+ }
+
for (Map.Entry<String, List<Pair<String, Integer>>> partitionEntry : partitionToFilesNameLengthMap.entrySet()) {
String partitionName = partitionEntry.getKey();
List<Pair<String, Integer>> fileNameAndLengthList = partitionEntry.getValue();
@@ -1343,6 +1360,10 @@ public class HoodieTestTable implements AutoCloseable {
public static List<HoodieWriteStat> generateHoodieWriteStatForPartition(Map<String, List<Pair<String, Integer>>> partitionToFileIdMap,
String commitTime, boolean bootstrap) {
List<HoodieWriteStat> writeStats = new ArrayList<>();
+ if (partitionToFileIdMap == null || partitionToFileIdMap.isEmpty()) {
+ return writeStats;
+ }
+
for (Map.Entry<String, List<Pair<String, Integer>>> entry : partitionToFileIdMap.entrySet()) {
String partition = entry.getKey();
for (Pair<String, Integer> fileIdInfo : entry.getValue()) {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
index e01bbf4f9e3..1d085c006d7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
@@ -19,6 +19,7 @@
package org.apache.hudi.table.action.compact;
+import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.common.config.HoodieMemoryConfig;
@@ -28,10 +29,13 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
@@ -47,6 +51,7 @@ import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.compact.strategy.PartitionRegexBasedCompactionStrategy;
+import org.apache.hudi.table.action.compact.strategy.SmallBoundedIOCompactionStrategy;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import com.codahale.metrics.Counter;
@@ -261,7 +266,7 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
.build();
try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
- String newCommitTime = "100";
+ String newCommitTime = writeClient.createNewInstantTime();
writeClient.startCommitWithTime(newCommitTime);
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
@@ -271,12 +276,12 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
// trigger 2 updates following with compaction
for (int i = 1; i < 5; i += 2) {
// Update all the 100 records
- newCommitTime = "10" + i;
+ newCommitTime = writeClient.createNewInstantTime();
updateRecords(config, newCommitTime, records);
assertLogFilesNumEqualsTo(config, 1);
- HoodieWriteMetadata result = compact(writeClient, "10" + (i + 1));
+ HoodieWriteMetadata result = compact(writeClient, writeClient.createNewInstantTime());
verifyCompaction(result);
// Verify compaction.requested, compaction.completed metrics counts.
@@ -314,7 +319,7 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
.withCompactionConfig(builder.build())
.withMetricsConfig(getMetricsConfig()).build();
try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
- String newCommitTime = "100";
+ String newCommitTime = writeClient.createNewInstantTime();
writeClient.startCommitWithTime(newCommitTime);
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 10);
@@ -322,18 +327,19 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
writeClient.insert(recordsRDD, newCommitTime).collect();
// update 1 time
- newCommitTime = "101";
+ newCommitTime = writeClient.createNewInstantTime();
updateRecords(config, newCommitTime, records);
assertLogFilesNumEqualsTo(config, 1);
// schedule compaction
- boolean scheduled = writeClient.scheduleCompactionAtInstant("102", Option.empty());
+ String compactionInstant = writeClient.createNewInstantTime();
+ boolean scheduled = writeClient.scheduleCompactionAtInstant(compactionInstant, Option.empty());
if (expectedCompactedPartition.isEmpty()) {
assertFalse(scheduled);
return;
}
- HoodieWriteMetadata result = compact(writeClient, "102");
+ HoodieWriteMetadata result = compact(writeClient, compactionInstant);
assertTrue(result.getWriteStats().isPresent());
List<HoodieWriteStat> stats = (List<HoodieWriteStat>) result.getWriteStats().get();
@@ -344,6 +350,82 @@ public class TestHoodieCompactor extends HoodieSparkClientTestHarness {
}
}
+ @Test
+ public void testPartitionsForIncrCompaction() throws Exception {
+ HoodieWriteConfig config = getConfigBuilder()
+ .withIncrementalTableServiceEnabled(true)
+ .withCompactionConfig(HoodieCompactionConfig
+ .newBuilder()
+ .withCompactionStrategy(new SmallBoundedIOCompactionStrategy())
+ .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+ .build();
+ String[] partitions = {"20250115"};
+
+ HoodieWriteConfig config2 = getConfigBuilder()
+ .withIncrementalTableServiceEnabled(true)
+ .withCompactionConfig(HoodieCompactionConfig
+ .newBuilder()
+ .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+ .build();
+ String[] partitions2 = {"20250116"};
+
+ try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
+ prepareRecords(writeClient, config, partitions);
+ prepareRecords(writeClient, config, partitions);
+ compact(writeClient, writeClient.createNewInstantTime());
+ HoodieCompactionPlan compactionPlan1 = getLatestCompactionPlan();
+
+ List<String> affectedPartitions = compactionPlan1.getOperations().stream()
+ .map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+ // compaction including 20250115
+ assertTrue(affectedPartitions.contains(partitions[0]));
+ List<String> missingPartitions = compactionPlan1.getMissingSchedulePartitions();
+ // compaction missing 20250115 because not all the fileSlices are all processed.
+ assertTrue(missingPartitions.contains(partitions[0]));
+ }
+
+ try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config2)) {
+ prepareRecords(writeClient, config2, partitions2);
+ prepareRecords(writeClient, config2, partitions2);
+ compact(writeClient, writeClient.createNewInstantTime());
+ HoodieCompactionPlan compactionPlan2 = getLatestCompactionPlan();
+ List<String> affectedPartitions2 = compactionPlan2.getOperations().stream()
+ .map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+ // compaction including 20250115 (fetched from recorded missing partitions)
+ assertTrue(affectedPartitions2.contains(partitions[0]));
+ // compaction including 20250116 (fetched from commit meta)
+ assertTrue(affectedPartitions2.contains(partitions2[0]));
+ // no missing partitions because all the fileSlices are all processed.
+ List<String> missingPartitions2 = compactionPlan2.getMissingSchedulePartitions();
+ assertTrue(missingPartitions2.isEmpty());
+ }
+ }
+
+ private void prepareRecords(SparkRDDWriteClient writeClient, HoodieWriteConfig config, String[] partitions) throws Exception {
+ initTestDataGenerator(partitions);
+ String newCommitTime = writeClient.createNewInstantTime();
+ writeClient.startCommitWithTime(newCommitTime);
+
+ // insert
+ List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
+ JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1);
+ writeClient.insert(recordsRDD, newCommitTime).collect();
+
+ // update
+ newCommitTime = writeClient.createNewInstantTime();
+ updateRecords(config, newCommitTime, records);
+ }
+
+ private HoodieCompactionPlan getLatestCompactionPlan() {
+ metaClient.reloadActiveTimeline();
+ Option<HoodieInstant> latestCompactionInstant = metaClient.getActiveTimeline().getCommitTimeline().lastInstant();
+ HoodieInstant compactionPlanInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION,
+ latestCompactionInstant.get().requestedTime(), InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+ Option<byte[]> details = metaClient.getActiveTimeline().readCompactionPlanAsBytes(compactionPlanInstant);
+
+ return CompactionUtils.getCompactionPlan(metaClient, details);
+ }
+
@Override
protected HoodieTableType getTableType() {
return HoodieTableType.MERGE_ON_READ;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java
new file mode 100644
index 00000000000..06ca7f24819
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/strategy/SmallBoundedIOCompactionStrategy.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.table.action.compact.strategy;
+
+import org.apache.hudi.avro.model.HoodieCompactionOperation;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class SmallBoundedIOCompactionStrategy extends BoundedIOCompactionStrategy {
+
+ // Compare to BoundedIOCompactionStrategy, SmallBoundedIOCompactionStrategy is limit targetIO to 0.1M
+ private final double targetIO = 0.1;
+ private final long opIo = 1;
+ public SmallBoundedIOCompactionStrategy() {
+ super();
+ }
+
+ @Override
+ public Pair<List<HoodieCompactionOperation>, List<String>> orderAndFilter(HoodieWriteConfig writeConfig,
+ List<HoodieCompactionOperation> operations,
+ List<HoodieCompactionPlan> pendingCompactionPlans) {
+ ArrayList<String> missingPartitions = new ArrayList<>();
+ List<HoodieCompactionOperation> finalOperations = new ArrayList<>();
+ double targetIORemaining = targetIO;
+ for (HoodieCompactionOperation op : operations) {
+ if (targetIORemaining > 0) {
+ targetIORemaining -= opIo;
+ finalOperations.add(op);
+ } else if (writeConfig.isIncrementalTableServiceEnabled()) {
+ missingPartitions.add(op.getPartitionPath());
+ } else {
+ return Pair.of(finalOperations, Collections.emptyList());
+ }
+ }
+ return Pair.of(finalOperations, missingPartitions);
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
index 6a43205128e..ff508c2ad50 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
@@ -486,6 +486,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
withTempDir { tmp =>
val tableName = generateTableName
val basePath = s"${tmp.getCanonicalPath}/$tableName"
+ val config = Map("hoodie.avro.schema.validate"-> "false")
spark.sql(
s"""
@@ -505,7 +506,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
| location '$basePath'
""".stripMargin)
- writeRecords(2, 4, 0, basePath, Map("hoodie.avro.schema.validate"-> "false"))
+ writeRecords(2, 4, 0, basePath, config)
val conf = new Configuration
val metaClient = HoodieTestUtils.createMetaClient(new HadoopStorageConfiguration(conf), basePath)
assert(0 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
@@ -516,16 +517,19 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
assert(0 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
assert(1 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
+ writeRecords(2, 4, 0, basePath, config)
spark.sql(s"call run_clustering(table => '$tableName', op => 'execute')")
metaClient.reloadActiveTimeline()
assert(1 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
assert(0 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
+ writeRecords(2, 4, 0, basePath, config)
spark.sql(s"call run_clustering(table => '$tableName')")
metaClient.reloadActiveTimeline()
assert(2 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
assert(0 == metaClient.getActiveTimeline.filterPendingClusteringTimeline().getInstants.size())
+ writeRecords(2, 4, 0, basePath, config)
spark.sql(s"call run_clustering(table => '$tableName')")
metaClient.reloadActiveTimeline()
assert(3 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
@@ -574,9 +578,10 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
val fileNum = 20
val numRecords = 400000
+ val config = metadataOpts ++ Map("hoodie.avro.schema.validate" -> "false")
// insert records
- writeRecords(fileNum, numRecords, 0, basePath, metadataOpts ++ Map("hoodie.avro.schema.validate"-> "false"))
+ val records = writeRecords(fileNum, numRecords, 0, basePath, config)
val conf = new Configuration
val metaClient = HoodieTestUtils.createMetaClient(new HadoopStorageConfiguration(conf), basePath)
val avgSize = avgRecord(metaClient.getActiveTimeline)
@@ -598,7 +603,11 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
val orderAllFiles = fileIndex1.allBaseFiles.size
val c2OrderFilterCount = fileIndex1.listFiles(Seq(), Seq(dataFilterC2)).head.files.size
val c3OrderFilterCount = fileIndex1.listFiles(Seq(), Seq(dataFilterC3)).head.files.size
+ val fs = new Path(basePath).getFileSystem(spark.sessionState.newHadoopConf())
+ // re-create and ingestion + clustering
+ fs.delete(new Path(basePath), true)
+ writeRecordsArray(fileNum, records, basePath, config ++ Map("hoodie.table.name" -> tableName))
spark.sql(
s"""call run_clustering(table => '$tableName', order => 'c2,c3', order_strategy => 'z-order', options => "
| hoodie.copyonwrite.record.size.estimate=$avgSize,
@@ -788,7 +797,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
Math.ceil((1.0 * totalByteSize) / totalRecordsCount).toLong
}
- def writeRecords(files: Int, numRecords: Int, partitions: Int, location: String, options: Map[String, String]): Unit = {
+ def writeRecords(files: Int, numRecords: Int, partitions: Int, location: String, options: Map[String, String]): util.ArrayList[Row] = {
val records = new util.ArrayList[Row](numRecords)
val rowDimension = Math.ceil(Math.sqrt(numRecords)).toInt
@@ -811,6 +820,19 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
StructField("c3", DataTypes.StringType, nullable = true, Metadata.empty)
))
+ // files can not effect for hudi
+ val df = spark.createDataFrame(records, struct).repartition(files)
+ writeDF(df, location, options)
+ records
+ }
+
+ def writeRecordsArray(files: Int, records: util.ArrayList[Row], location: String, options: Map[String, String]): Unit = {
+ val struct = StructType(Array[StructField](
+ StructField("c1", DataTypes.IntegerType, nullable = true, Metadata.empty),
+ StructField("c2", DataTypes.StringType, nullable = true, Metadata.empty),
+ StructField("c3", DataTypes.StringType, nullable = true, Metadata.empty)
+ ))
+
// files can not effect for hudi
val df = spark.createDataFrame(records, struct).repartition(files)
writeDF(df, location, options)