You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/14 22:08:56 UTC
(hudi) 20/28: [HUDI-7531] Consider pending clustering when scheduling a new clustering plan (#10923)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9de9cbb66b777ebf3ccd3be473175c4f6e285f13
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Tue May 14 15:07:35 2024 -0700
[HUDI-7531] Consider pending clustering when scheduling a new clustering plan (#10923)
---
.../PreferWriterConflictResolutionStrategy.java | 2 +-
.../cluster/ClusteringPlanActionExecutor.java | 3 +-
.../rollback/BaseRollbackActionExecutor.java | 2 +-
.../action/rollback/RestorePlanActionExecutor.java | 2 +-
.../table/timeline/HoodieDefaultTimeline.java | 20 +---
.../hudi/common/table/timeline/HoodieTimeline.java | 3 +-
.../apache/hudi/common/util/ClusteringUtils.java | 44 ++++++---
.../hudi/common/util/TestClusteringUtils.java | 2 +
.../java/org/apache/hudi/util/StreamerUtil.java | 3 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 101 +++++++++++++++++----
10 files changed, 128 insertions(+), 54 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/PreferWriterConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/PreferWriterConflictResolutionStrategy.java
index f95e7b078a6..3fd0a836915 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/PreferWriterConflictResolutionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/PreferWriterConflictResolutionStrategy.java
@@ -55,7 +55,7 @@ public class PreferWriterConflictResolutionStrategy
Option<HoodieInstant> lastSuccessfulInstant) {
HoodieActiveTimeline activeTimeline = metaClient.reloadActiveTimeline();
if ((REPLACE_COMMIT_ACTION.equals(currentInstant.getAction())
- && ClusteringUtils.isClusteringCommit(metaClient, currentInstant))
+ && ClusteringUtils.isClusteringInstant(activeTimeline, currentInstant))
|| COMPACTION_ACTION.equals(currentInstant.getAction())) {
return getCandidateInstantsForTableServicesCommits(activeTimeline, currentInstant);
} else {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
index b8c38bd140d..54df15d6e80 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanActionExecutor.java
@@ -57,7 +57,8 @@ public class ClusteringPlanActionExecutor<T, I, K, O> extends BaseActionExecutor
protected Option<HoodieClusteringPlan> createClusteringPlan() {
LOG.info("Checking if clustering needs to be run on " + config.getBasePath());
- Option<HoodieInstant> lastClusteringInstant = table.getActiveTimeline().getLastClusterCommit();
+ Option<HoodieInstant> lastClusteringInstant =
+ table.getActiveTimeline().getLastClusteringInstant();
int commitsSinceLastClustering = table.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()
.findInstantsAfter(lastClusteringInstant.map(HoodieInstant::getTimestamp).orElse("0"), Integer.MAX_VALUE)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index f2a40512b88..d41120e68dc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -191,7 +191,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
if (!instant.getAction().equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
return true;
}
- return !ClusteringUtils.isPendingClusteringInstant(table.getMetaClient(), instant);
+ return !ClusteringUtils.isClusteringInstant(table.getActiveTimeline(), instant);
}).map(HoodieInstant::getTimestamp)
.collect(Collectors.toList());
if ((instantTimeToRollback != null) && !inflights.isEmpty()
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RestorePlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RestorePlanActionExecutor.java
index b3ee11b9836..2f9e96859ff 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RestorePlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RestorePlanActionExecutor.java
@@ -71,7 +71,7 @@ public class RestorePlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T,
// rollback pending clustering instants first before other instants (See HUDI-3362)
List<HoodieInstant> pendingClusteringInstantsToRollback = table.getActiveTimeline().filterPendingReplaceTimeline()
// filter only clustering related replacecommits (Not insert_overwrite related commits)
- .filter(instant -> ClusteringUtils.isPendingClusteringInstant(table.getMetaClient(), instant))
+ .filter(instant -> ClusteringUtils.isClusteringInstant(table.getActiveTimeline(), instant))
.getReverseOrderedInstants()
.filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), savepointToRestoreTimestamp))
.collect(Collectors.toList());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
index e3c468919fe..a26bed061d6 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
@@ -18,8 +18,6 @@
package org.apache.hudi.common.table.timeline;
-import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.ClusteringUtils;
import org.apache.hudi.common.util.CollectionUtils;
@@ -30,7 +28,6 @@ import org.apache.hudi.exception.HoodieException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -499,25 +496,18 @@ public class HoodieDefaultTimeline implements HoodieTimeline {
}
@Override
- public Option<HoodieInstant> getLastClusterCommit() {
- return Option.fromJavaOptional(getCommitsTimeline().filter(s -> s.getAction().equalsIgnoreCase(HoodieTimeline.REPLACE_COMMIT_ACTION))
+ public Option<HoodieInstant> getLastClusteringInstant() {
+ return Option.fromJavaOptional(getCommitsTimeline().filter(s -> s.getAction().equalsIgnoreCase(HoodieTimeline.REPLACE_COMMIT_ACTION))
.getReverseOrderedInstants()
- .filter(i -> {
- try {
- HoodieCommitMetadata metadata = TimelineUtils.getCommitMetadata(i, this);
- return metadata.getOperationType().equals(WriteOperationType.CLUSTER);
- } catch (IOException e) {
- LOG.warn("Unable to read commit metadata for " + i + " due to " + e.getMessage());
- return false;
- }
- }).findFirst());
+ .filter(i -> ClusteringUtils.isClusteringInstant(this, i))
+ .findFirst());
}
@Override
public Option<HoodieInstant> getLastPendingClusterInstant() {
return Option.fromJavaOptional(filterPendingReplaceTimeline()
.getReverseOrderedInstants()
- .filter(i -> ClusteringUtils.isPendingClusteringInstant(this, i)).findFirst());
+ .filter(i -> ClusteringUtils.isClusteringInstant(this, i)).findFirst());
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
index 11979a2c9e8..cdbe5b15fc5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
@@ -397,9 +397,8 @@ public interface HoodieTimeline extends Serializable {
/**
* get the most recent cluster commit if present
- *
*/
- public Option<HoodieInstant> getLastClusterCommit();
+ public Option<HoodieInstant> getLastClusteringInstant();
/**
* get the most recent pending cluster commit if present
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
index 6fe46c6c109..50c76e7ed64 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
@@ -35,6 +35,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
@@ -74,15 +75,22 @@ public class ClusteringUtils {
}
/**
- * Checks if the replacecommit is clustering commit.
+ * Checks if the requested, inflight, or completed instant of replacecommit action
+ * is a clustering operation, by checking whether the requested instant contains
+ * a clustering plan.
+ *
+ * @param timeline Hudi timeline.
+ * @param replaceInstant the instant of replacecommit action to check.
+ * @return whether the instant is a clustering operation.
*/
- public static boolean isClusteringCommit(HoodieTableMetaClient metaClient, HoodieInstant pendingReplaceInstant) {
- return getClusteringPlan(metaClient, pendingReplaceInstant).isPresent();
+ public static boolean isClusteringInstant(HoodieTimeline timeline, HoodieInstant replaceInstant) {
+ return getClusteringPlan(timeline, replaceInstant).isPresent();
}
/**
* Get requested replace metadata from timeline.
- * @param timeline used to get the bytes stored in the requested replace instant in the timeline
+ *
+ * @param timeline used to get the bytes stored in the requested replace instant in the timeline
* @param pendingReplaceInstant can be in any state, because it will always be converted to requested state
* @return option of the replace metadata if present, else empty
* @throws IOException
@@ -237,16 +245,8 @@ public class ClusteringUtils {
public static List<HoodieInstant> getPendingClusteringInstantTimes(HoodieTableMetaClient metaClient) {
return metaClient.getActiveTimeline().filterPendingReplaceTimeline().getInstantsAsStream()
- .filter(instant -> isPendingClusteringInstant(metaClient, instant))
- .collect(Collectors.toList());
- }
-
- public static boolean isPendingClusteringInstant(HoodieTableMetaClient metaClient, HoodieInstant instant) {
- return getClusteringPlan(metaClient, instant).isPresent();
- }
-
- public static boolean isPendingClusteringInstant(HoodieTimeline timeline, HoodieInstant instant) {
- return getClusteringPlan(timeline, instant).isPresent();
+ .filter(instant -> isClusteringInstant(metaClient.getActiveTimeline(), instant))
+ .collect(Collectors.toList());
}
/**
@@ -311,4 +311,20 @@ public class ClusteringUtils {
}
return oldestInstantToRetain;
}
+
+ /**
+ * @param instant Hudi instant to check.
+ * @param timeline Hudi timeline.
+ * @return whether the given {@code instant} is a completed clustering operation.
+ */
+ public static boolean isCompletedClusteringInstant(HoodieInstant instant, HoodieTimeline timeline) {
+ if (!instant.getAction().equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
+ return false;
+ }
+ try {
+ return TimelineUtils.getCommitMetadata(instant, timeline).getOperationType().equals(WriteOperationType.CLUSTER);
+ } catch (IOException e) {
+ throw new HoodieException("Resolve replace commit metadata error for instant: " + instant, e);
+ }
+ }
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java
index 513b352620a..2fa676bbb41 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java
@@ -133,7 +133,9 @@ public class TestClusteringUtils extends HoodieCommonTestHarness {
String clusterTime1 = "1";
HoodieInstant requestedInstant = createRequestedReplaceInstant(partitionPath1, clusterTime1, fileIds1);
HoodieInstant inflightInstant = metaClient.getActiveTimeline().transitionReplaceRequestedToInflight(requestedInstant, Option.empty());
+ assertTrue(ClusteringUtils.isClusteringInstant(metaClient.getActiveTimeline(), requestedInstant));
HoodieClusteringPlan requestedClusteringPlan = ClusteringUtils.getClusteringPlan(metaClient, requestedInstant).get().getRight();
+ assertTrue(ClusteringUtils.isClusteringInstant(metaClient.getActiveTimeline(), inflightInstant));
HoodieClusteringPlan inflightClusteringPlan = ClusteringUtils.getClusteringPlan(metaClient, inflightInstant).get().getRight();
assertEquals(requestedClusteringPlan, inflightClusteringPlan);
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java
index 672c3fd2526..d83012f6bc7 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.ClusteringUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
@@ -462,7 +463,7 @@ public class StreamerUtil {
public static boolean isWriteCommit(HoodieTableType tableType, HoodieInstant instant, HoodieTimeline timeline) {
return tableType == HoodieTableType.MERGE_ON_READ
? !instant.getAction().equals(HoodieTimeline.COMMIT_ACTION) // not a compaction
- : !ClusteringUtil.isClusteringInstant(instant, timeline); // not a clustering
+ : !ClusteringUtils.isCompletedClusteringInstant(instant, timeline); // not a clustering
}
/**
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index 22a61d58881..e2e0cf087dd 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -17,15 +17,14 @@
package org.apache.hudi.functional
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
import org.apache.hudi.DataSourceWriteOptions.{INLINE_CLUSTERING_ENABLE, KEYGENERATOR_CLASS_NAME}
import org.apache.hudi.HoodieConversionUtils.toJavaOption
import org.apache.hudi.QuickstartUtils.{convertToStringList, getQuickstartWriteConfigs}
import org.apache.hudi.avro.AvroSchemaCompatibility.SchemaIncompatibilityType
+import org.apache.hudi.client.SparkRDDWriteClient
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.config.{HoodieCommonConfig, HoodieMetadataConfig}
import org.apache.hudi.common.config.TimestampKeyGeneratorConfig.{TIMESTAMP_INPUT_DATE_FORMAT, TIMESTAMP_OUTPUT_DATE_FORMAT, TIMESTAMP_TIMEZONE_FORMAT, TIMESTAMP_TYPE_FIELD}
+import org.apache.hudi.common.config.{HoodieCommonConfig, HoodieMetadataConfig}
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType}
@@ -33,7 +32,8 @@ import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline, Tim
import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.{deleteRecordsToStrings, recordsToStrings}
-import org.apache.hudi.common.util
+import org.apache.hudi.common.util.{ClusteringUtils, Option}
+import org.apache.hudi.common.{HoodiePendingRollbackInfo, util}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.config.metrics.HoodieMetricsConfig
import org.apache.hudi.exception.ExceptionUtil.getRootCause
@@ -44,9 +44,13 @@ import org.apache.hudi.hive.HiveSyncConfigHolder
import org.apache.hudi.keygen._
import org.apache.hudi.keygen.constant.KeyGeneratorOptions
import org.apache.hudi.metrics.{Metrics, MetricsReporterType}
+import org.apache.hudi.table.HoodieSparkTable
import org.apache.hudi.testutils.HoodieSparkClientTestBase
import org.apache.hudi.util.JFunction
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers, QuickstartUtils, ScalaAssertionSupport}
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hudi.HoodieSparkSessionExtension
@@ -62,6 +66,7 @@ import org.junit.jupiter.params.provider.{CsvSource, EnumSource, ValueSource}
import java.sql.{Date, Timestamp}
import java.util.concurrent.{CountDownLatch, TimeUnit}
import java.util.function.Consumer
+
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.util.matching.Regex
@@ -1819,9 +1824,9 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
}
@ParameterizedTest
- @EnumSource(value = classOf[HoodieRecordType], names = Array("AVRO", "SPARK"))
- def testInsertOverwriteCluster(recordType: HoodieRecordType): Unit = {
- val (writeOpts, _) = getWriterReaderOpts(recordType)
+ @EnumSource(value = classOf[HoodieInstant.State], names = Array("REQUESTED", "INFLIGHT", "COMPLETED"))
+ def testInsertOverwriteCluster(firstClusteringState: HoodieInstant.State): Unit = {
+ val (writeOpts, _) = getWriterReaderOpts()
// Insert Operation
val records = recordsToStrings(dataGen.generateInserts("000", 100)).toList
@@ -1831,6 +1836,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
INLINE_CLUSTERING_ENABLE.key() -> "true",
"hoodie.clustering.inline.max.commits" -> "2",
"hoodie.clustering.plan.strategy.sort.columns" -> "_row_key",
+ "hoodie.clustering.plan.strategy.max.num.groups" -> "1",
"hoodie.insert.shuffle.parallelism" -> "4",
"hoodie.upsert.shuffle.parallelism" -> "4",
DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key",
@@ -1843,7 +1849,15 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Overwrite)
.save(basePath)
- for (i <- 1 until 6) {
+ val metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(hadoopConf)
+ .build()
+
+ assertFalse(metaClient.getActiveTimeline.getLastClusteringInstant.isPresent)
+
+ var lastClustering: HoodieInstant = null
+ for (i <- 1 until 4) {
val records = recordsToStrings(dataGen.generateInsertsForPartition("00" + i, 10, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).toList
val inputDF = spark.read.json(spark.sparkContext.parallelize(records, 2))
inputDF.write.format("hudi")
@@ -1851,21 +1865,72 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OVERWRITE_OPERATION_OPT_VAL)
.mode(SaveMode.Append)
.save(basePath)
+ val lastInstant = metaClient.reloadActiveTimeline.getCommitsTimeline.lastInstant.get
+ if (i == 1 || i == 3) {
+ // Last instant is clustering
+ assertTrue(TimelineUtils.getCommitMetadata(lastInstant, metaClient.getActiveTimeline)
+ .getOperationType.equals(WriteOperationType.CLUSTER))
+ assertTrue(ClusteringUtils.isClusteringInstant(metaClient.getActiveTimeline, lastInstant))
+ lastClustering = lastInstant
+ assertEquals(
+ lastClustering,
+ metaClient.getActiveTimeline.getLastClusteringInstant.get)
+ } else {
+ assertTrue(TimelineUtils.getCommitMetadata(lastInstant, metaClient.getActiveTimeline)
+ .getOperationType.equals(WriteOperationType.INSERT_OVERWRITE))
+ assertFalse(ClusteringUtils.isClusteringInstant(metaClient.getActiveTimeline, lastInstant))
+ assertEquals(
+ lastClustering,
+ metaClient.getActiveTimeline.getLastClusteringInstant.get)
+ }
+ if (i == 1) {
+ val writeConfig = HoodieWriteConfig.newBuilder()
+ .forTable("hoodie_test")
+ .withPath(basePath)
+ .withProps(optsWithCluster)
+ .build()
+ if (firstClusteringState == HoodieInstant.State.INFLIGHT
+ || firstClusteringState == HoodieInstant.State.REQUESTED) {
+ // Move the clustering to inflight for testing
+ fs.delete(new Path(metaClient.getMetaPath, lastInstant.getFileName), false)
+ val inflightClustering = metaClient.reloadActiveTimeline.lastInstant.get
+ assertTrue(inflightClustering.isInflight)
+ assertEquals(
+ inflightClustering,
+ metaClient.getActiveTimeline.getLastClusteringInstant.get)
+ }
+ if (firstClusteringState == HoodieInstant.State.REQUESTED) {
+ val table = HoodieSparkTable.create(writeConfig, context)
+ table.rollbackInflightClustering(
+ metaClient.getActiveTimeline.getLastClusteringInstant.get,
+ new java.util.function.Function[String, Option[HoodiePendingRollbackInfo]] {
+ override def apply(commitToRollback: String): Option[HoodiePendingRollbackInfo] = {
+ new SparkRDDWriteClient(context, writeConfig).getTableServiceClient
+ .getPendingRollbackInfo(table.getMetaClient, commitToRollback, false)
+ }
+ })
+ val requestedClustering = metaClient.reloadActiveTimeline.getCommitsTimeline.lastInstant.get
+ assertTrue(requestedClustering.isRequested)
+ assertEquals(
+ requestedClustering,
+ metaClient.getActiveTimeline.getLastClusteringInstant.get)
+ }
+ // This should not schedule any new clustering
+ new SparkRDDWriteClient(context, writeConfig)
+ .scheduleClustering(org.apache.hudi.common.util.Option.of(Map[String, String]()))
+ assertEquals(lastInstant.getTimestamp,
+ metaClient.reloadActiveTimeline.getCommitsTimeline.lastInstant.get.getTimestamp)
+ }
}
-
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(hadoopConf)
- .build()
- val timeline = metaClient.getActiveTimeline
- val instants = timeline.getAllCommitsTimeline.filterCompletedInstants.getInstants
- assertEquals(9, instants.size)
+ val timeline = metaClient.reloadActiveTimeline
+ val instants = timeline.getCommitsTimeline.getInstants
+ assertEquals(6, instants.size)
val replaceInstants = instants.filter(i => i.getAction.equals(HoodieTimeline.REPLACE_COMMIT_ACTION)).toList
- assertEquals(8, replaceInstants.size)
+ assertEquals(5, replaceInstants.size)
val clusterInstants = replaceInstants.filter(i => {
TimelineUtils.getCommitMetadata(i, metaClient.getActiveTimeline).getOperationType.equals(WriteOperationType.CLUSTER)
})
- assertEquals(3, clusterInstants.size)
+ assertEquals(2, clusterInstants.size)
}