You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/08/09 14:48:32 UTC
[hudi] 32/48: [HUDI-4534] Fixing upgrade to reload Metaclient for deltastreamer writes (#6296)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch release-0.12.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b124f5e4871eedc362207f036520fab71939b6b4
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Sat Aug 6 23:08:24 2022 -0400
[HUDI-4534] Fixing upgrade to reload Metaclient for deltastreamer writes (#6296)
---
.../main/java/org/apache/hudi/client/BaseHoodieWriteClient.java | 9 ++++++---
.../apache/hudi/client/functional/TestHoodieBackedMetadata.java | 1 +
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 4cf58f1c30..48dd21d28e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -1553,13 +1553,16 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper);
if (upgradeDowngrade.needsUpgradeOrDowngrade(HoodieTableVersion.current())) {
+ metaClient = HoodieTableMetaClient.reload(metaClient);
// Ensure no inflight commits by setting EAGER policy and explicitly cleaning all failed commits
List<String> instantsToRollback = getInstantsToRollback(metaClient, HoodieFailedWritesCleaningPolicy.EAGER, instantTime);
- Map<String, Option<HoodiePendingRollbackInfo>> pendingRollbacks = getPendingRollbackInfos(metaClient);
- instantsToRollback.forEach(entry -> pendingRollbacks.putIfAbsent(entry, Option.empty()));
+ if (!instantsToRollback.isEmpty()) {
+ Map<String, Option<HoodiePendingRollbackInfo>> pendingRollbacks = getPendingRollbackInfos(metaClient);
+ instantsToRollback.forEach(entry -> pendingRollbacks.putIfAbsent(entry, Option.empty()));
- rollbackFailedWrites(pendingRollbacks, true);
+ rollbackFailedWrites(pendingRollbacks, true);
+ }
new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper)
.run(HoodieTableVersion.current(), instantTime.orElse(null));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 1058ccec60..a3f9b639af 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -2082,6 +2082,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
.withLockConfig(HoodieLockConfig.newBuilder().withLockProvider(InProcessLockProvider.class).build())
.withProperties(properties)
+ .withEmbeddedTimelineServerEnabled(false)
.build();
// With next commit the table should be re-bootstrapped and partial commit should be rolled back.