You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2023/06/19 02:32:09 UTC
[hudi] branch master updated: [HUDI-6401] Should not throw exception when create marker file for log file (#9003)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ea723504fbd [HUDI-6401] Should not throw exception when create marker file for log file (#9003)
ea723504fbd is described below
commit ea723504fbdb760fd00b4505fcbc737b4176370f
Author: guanziyue <30...@users.noreply.github.com>
AuthorDate: Mon Jun 19 10:32:03 2023 +0800
[HUDI-6401] Should not throw exception when create marker file for log file (#9003)
---
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 3 +--
.../org/apache/hudi/table/marker/WriteMarkers.java | 28 ++++++++++++++++++++++
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index 4d5c8baefad..6653d348882 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -294,9 +294,8 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
@Override
public boolean preLogFileOpen(HoodieLogFile logFileToAppend) {
- // we use create rather than createIfNotExists because create method can trigger marker-based early conflict detection.
WriteMarkers writeMarkers = WriteMarkersFactory.get(config.getMarkersType(), hoodieTable, instantTime);
- return writeMarkers.create(partitionPath, logFileToAppend.getFileName(), IOType.APPEND,
+ return writeMarkers.createIfNotExists(partitionPath, logFileToAppend.getFileName(), IOType.APPEND,
config, fileId, hoodieTable.getMetaClient().getActiveTimeline()).isPresent();
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
index bbbb2cdfd90..8a9c5c4ad1a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
@@ -107,6 +107,34 @@ public abstract class WriteMarkers implements Serializable {
return create(partitionPath, fileName, type, true);
}
+ /**
+ * Creates a marker if the marker does not exist.
+ * This can invoke marker-based early conflict detection when enabled for multi-writers.
+ *
+ * @param partitionPath partition path in the table
+ * @param fileName file name
+ * @param type write IO type
+ * @param writeConfig Hudi write configs.
+ * @param fileId File ID.
+ * @param activeTimeline Active timeline for the write operation.
+ * @return the marker path.
+ */
+ public Option<Path> createIfNotExists(String partitionPath, String fileName, IOType type, HoodieWriteConfig writeConfig,
+ String fileId, HoodieActiveTimeline activeTimeline) {
+ if (writeConfig.isEarlyConflictDetectionEnable()
+ && writeConfig.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
+ HoodieTimeline pendingCompactionTimeline = activeTimeline.filterPendingCompactionTimeline();
+ HoodieTimeline pendingReplaceTimeline = activeTimeline.filterPendingReplaceTimeline();
+ // TODO If current is compact or clustering then create marker directly without early conflict detection.
+ // Need to support early conflict detection between table service and common writers.
+ if (pendingCompactionTimeline.containsInstant(instantTime) || pendingReplaceTimeline.containsInstant(instantTime)) {
+ return create(partitionPath, fileName, type, true);
+ }
+ return createWithEarlyConflictDetection(partitionPath, fileName, type, false, writeConfig, fileId, activeTimeline);
+ }
+ return create(partitionPath, fileName, type, true);
+ }
+
/**
* Quietly deletes the marker directory.
*