(hbase) branch branch-2 updated: HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)

[ju...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

junegunn pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
     new 1cb250a769b HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)
1cb250a769b is described below

commit 1cb250a769bc80507750156fbe21ab95abf66bbe
Author: Junegunn Choi <ju...@apache.org>
AuthorDate: Wed Oct 21 08:39:46 2026 +0900

    HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)
    
    Signed-off-by: Andor Molnar <an...@apache.org>
---
 .../hadoop/hbase/mapreduce/WALInputFormat.java     |  5 +++
 .../hadoop/hbase/mapreduce/TestWALInputFormat.java | 39 ++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
index 2cd19d682ba..8f67045e38d 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
 import org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException;
 import org.apache.hadoop.hbase.util.LeaseNotRecoveredException;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -413,6 +414,10 @@ public class WALInputFormat extends InputFormat<WALKey, WALEdit> {
 
   static void addFile(List<FileStatus> result, FileSystem fs, LocatedFileStatus lfs, long startTime,
     long endTime) {
+    if (lfs.getPath().getName().endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX)) {
+      LOG.debug("Skipped master-local WAL {}", lfs.getPath());
+      return;
+    }
     long timestamp = WAL.getTimestamp(lfs.getPath().getName());
     if (timestamp > 0) {
       // The name carries the WAL's creation time, which only bounds its entries from below. A WAL
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
index 77de507f46f..83194502456 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
@@ -23,8 +23,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -98,6 +101,42 @@ public class TestWALInputFormat {
     assertEquals(9, lfss.size());
   }
 
+  @Test
+  public void testSkipsArchivedMasterLocalWALs() throws Exception {
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ",");
+    conf.setLong(WALInputFormat.START_TIME_KEY, 200);
+    conf.setLong(WALInputFormat.END_TIME_KEY, 300);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path inputDir = TEST_UTIL.getDataTestDirOnTestFS("master-local-wals");
+    Path nonEmpty = new Path(inputDir, "master.250$masterlocalwal$");
+    Path empty = new Path(inputDir, "nested/master.251$masterlocalwal$");
+    List<String> expected =
+      Arrays.asList("0000000000000016310", "server.250", "server.250.meta", "server.250.syncrep");
+    try {
+      for (String name : expected) {
+        try (FSDataOutputStream out = fs.create(new Path(inputDir, "nested/" + name))) {
+          out.write(1);
+        }
+      }
+      try (FSDataOutputStream out = fs.create(nonEmpty)) {
+        out.write(1);
+      }
+      fs.create(empty).close();
+
+      conf.set(FileInputFormat.INPUT_DIR, inputDir.toString());
+      List<String> actual = new WALInputFormat().getSplits(Job.getInstance(conf)).stream()
+        .map(split -> new Path(((WALInputFormat.WALSplit) split).getLogFileName()).getName())
+        .sorted().collect(Collectors.toList());
+      assertEquals(expected, actual);
+
+      conf.set(FileInputFormat.INPUT_DIR, nonEmpty + "," + empty);
+      assertTrue(new WALInputFormat().getSplits(Job.getInstance(conf)).isEmpty());
+    } finally {
+      fs.delete(inputDir, true);
+    }
+  }
+
   private static boolean isKept(FileSystem fs, long created, long mtime, long start, long end) {
     return isKept(fs, created, mtime, mtime, start, end);
   }