[hudi] branch master updated: [HUDI-5270] Duplicate key error when insert_overwrite the same partition in multi writer (#7281)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new a1da9ec2e4f [HUDI-5270] Duplicate key error when insert_overwrite the same partition in multi writer (#7281)
a1da9ec2e4f is described below

commit a1da9ec2e4fd87f2b1637bba8e239eaf1eeba3a6
Author: Ming Wei <29...@qq.com>
AuthorDate: Wed Feb 8 11:10:58 2023 +0800

    [HUDI-5270] Duplicate key error when insert_overwrite the same partition in multi writer (#7281)
    
    Co-authored-by: jameswei <ja...@didiglobal.com>
---
 .../apache/hudi/common/table/view/AbstractTableFileSystemView.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
index 8ea017977e9..bf4dad5deb3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
@@ -247,7 +247,9 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
       }
     });
 
-    Map<HoodieFileGroupId, HoodieInstant> replacedFileGroups = resultStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    // Duplicate key error when insert_overwrite same partition in multi writer, keep the instant with greater timestamp when the file group id conflicts
+    Map<HoodieFileGroupId, HoodieInstant> replacedFileGroups = resultStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
+        (instance1, instance2) -> HoodieTimeline.compareTimestamps(instance1.getTimestamp(), HoodieTimeline.LESSER_THAN, instance2.getTimestamp()) ? instance2 : instance1));
     resetReplacedFileGroups(replacedFileGroups);
     LOG.info("Took " + hoodieTimer.endTimer() + " ms to read  " + replacedTimeline.countInstants() + " instants, "
         + replacedFileGroups.size() + " replaced file groups");