You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/01/11 05:31:39 UTC
[hudi] 03/06: Removing rollbacks instants from timeline for restore operation (#4518)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.10.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b6fde073961b8ddb5a2175f8ed0d6ea59e5798a4
Author: Sivabalan Narayanan <si...@uber.com>
AuthorDate: Sun Jan 9 21:14:28 2022 -0500
Removing rollbacks instants from timeline for restore operation (#4518)
---
.../hudi/table/action/restore/BaseRestoreActionExecutor.java | 10 ++++++++++
.../functional/TestHoodieClientOnCopyOnWriteStorage.java | 2 ++
2 files changed, 12 insertions(+)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index 9371340..58247bb 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -99,6 +99,16 @@ public abstract class BaseRestoreActionExecutor<T extends HoodieRecordPayload, I
writeToMetadata(restoreMetadata);
table.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, instantTime),
TimelineMetadataUtils.serializeRestoreMetadata(restoreMetadata));
+ // get all rollbacks instants after restore instant time and delete them.
+ // if not, rollbacks will be considered not completed and might hinder metadata table compaction.
+ List<HoodieInstant> instantsToRollback = table.getActiveTimeline().getRollbackTimeline()
+ .getReverseOrderedInstants()
+ .filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), restoreInstantTime))
+ .collect(Collectors.toList());
+ instantsToRollback.forEach(entry -> {
+ table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
+ table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
+ });
LOG.info("Commits " + instantsRolledBack + " rollback is complete. Restored table to " + restoreInstantTime);
return restoreMetadata;
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index aa3ead4..2ec4f7d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -577,6 +577,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
client = getHoodieWriteClient(newConfig);
client.restoreToInstant("004");
+ assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());
+
// Check the entire dataset has all records still
String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
for (int i = 0; i < fullPartitionPaths.length; i++) {