You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2026/05/04 20:42:51 UTC

(hudi) branch master updated: refactor: move checkpoint metadata lookup helper to hudi-common (#18489)

This is an automated email from the ASF dual-hosted git repository.

nsivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 471bb48338bd refactor: move checkpoint metadata lookup helper to hudi-common (#18489)
471bb48338bd is described below

commit 471bb48338bd9642bd593472b029915180156038
Author: Surya Prasanna <sy...@uber.com>
AuthorDate: Mon May 4 13:42:43 2026 -0700

    refactor: move checkpoint metadata lookup helper to hudi-common (#18489)
    
    This PR moves the checkpoint metadata lookup helper into hudi-common so ingestion-related code can reuse the same timeline utility instead of keeping the logic in utilities-only code.
---
 .../hudi/common/table/timeline/TimelineUtils.java  | 27 ++++++++++++++++++++++
 .../streamer/StreamerCheckpointUtils.java          | 22 ++++++------------
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
index cfc1520b2393..4b2e6ee55fc6 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
@@ -49,6 +49,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.text.ParseException;
 import java.util.AbstractMap;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -647,4 +648,30 @@ public class TimelineUtils {
     }
     return writerOption;
   }
+
+  /**
+   * Returns the latest reverse-ordered instant whose commit metadata contains at least one of the provided
+   * checkpoint metadata keys.
+   *
+   * @param timeline timeline to scan; expected to contain completed instants
+   * @param checkpointKeys checkpoint metadata keys to look for in commit metadata
+   * @return an {@link Option} containing a {@link Pair} of the matching instant's
+   *         {@link HoodieInstant#toString()} value and its
+   *         {@link HoodieCommitMetadata}; {@link Option#empty()} if no matching instant is found
+   * @throws IOException if reading commit metadata fails
+   */
+  @SuppressWarnings("unchecked")
+  public static Option<Pair<String, HoodieCommitMetadata>> getLatestInstantAndCommitMetadataWithValidCheckpointInfo(
+      HoodieTimeline timeline, String... checkpointKeys) throws IOException {
+    return (Option<Pair<String, HoodieCommitMetadata>>) timeline.getReverseOrderedInstants().map(instant -> {
+      try {
+        HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(instant);
+        boolean hasCheckpointMetadata = Arrays.stream(checkpointKeys)
+            .anyMatch(key -> !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(key)));
+        return hasCheckpointMetadata ? Option.of(Pair.of(instant.toString(), commitMetadata)) : Option.empty();
+      } catch (IOException e) {
+        throw new HoodieIOException("Failed to parse HoodieCommitMetadata for " + instant.toString(), e);
+      }
+    }).filter(Option::isPresent).findFirst().orElse(Option.empty());
+  }
 }
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java
index 03db67b9d03f..560d9fd2172a 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamerCheckpointUtils.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.table.checkpoint.Checkpoint;
 import org.apache.hudi.common.table.checkpoint.CheckpointUtils;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
 import org.apache.hudi.common.util.ConfigUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
@@ -201,21 +202,12 @@ public class StreamerCheckpointUtils {
 
   public static Option<Pair<String, HoodieCommitMetadata>> getLatestInstantAndCommitMetadataWithValidCheckpointInfo(HoodieTimeline timeline)
       throws IOException {
-    return (Option<Pair<String, HoodieCommitMetadata>>) timeline.getReverseOrderedInstants().map(instant -> {
-      try {
-        HoodieCommitMetadata commitMetadata = timeline.readCommitMetadata(instant);
-        if (!StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_KEY))
-            || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieStreamer.CHECKPOINT_RESET_KEY))
-            || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(STREAMER_CHECKPOINT_KEY_V2))
-            || !StringUtils.isNullOrEmpty(commitMetadata.getMetadata(STREAMER_CHECKPOINT_RESET_KEY_V2))) {
-          return Option.of(Pair.of(instant.toString(), commitMetadata));
-        } else {
-          return Option.empty();
-        }
-      } catch (IOException e) {
-        throw new HoodieIOException("Failed to parse HoodieCommitMetadata for " + instant.toString(), e);
-      }
-    }).filter(Option::isPresent).findFirst().orElse(Option.empty());
+    return TimelineUtils.getLatestInstantAndCommitMetadataWithValidCheckpointInfo(
+        timeline,
+        HoodieStreamer.CHECKPOINT_KEY,
+        HoodieStreamer.CHECKPOINT_RESET_KEY,
+        STREAMER_CHECKPOINT_KEY_V2,
+        STREAMER_CHECKPOINT_RESET_KEY_V2);
   }
 
   public static Option<HoodieCommitMetadata> getLatestCommitMetadataWithValidCheckpointInfo(HoodieTimeline timeline) throws IOException {