You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ju...@apache.org on 2026/10/20 23:39:53 UTC

(hbase) branch master updated: HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)

This is an automated email from the ASF dual-hosted git repository.

junegunn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
     new e4edde9a81e HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)
e4edde9a81e is described below

commit e4edde9a81efd4a5db5057bf5847e202ba19bf2e
Author: Junegunn Choi <ju...@apache.org>
AuthorDate: Wed Oct 21 08:39:46 2026 +0900

    HBASE-30440 Skip archived master-local WALs in WALInputFormat (#8720)
    
    Signed-off-by: Andor Molnar <an...@apache.org>
---
 .../hadoop/hbase/mapreduce/WALInputFormat.java     |  5 +++
 .../hadoop/hbase/mapreduce/TestWALInputFormat.java | 42 ++++++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
index 0146ff532d4..1250b93320f 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
 import org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException;
 import org.apache.hadoop.hbase.util.LeaseNotRecoveredException;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -420,6 +421,10 @@ public class WALInputFormat extends InputFormat<WALKey, WALEdit> {
 
   static void addFile(List<FileStatus> result, FileSystem fs, LocatedFileStatus lfs, long startTime,
     long endTime) {
+    if (lfs.getPath().getName().endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX)) {
+      LOG.debug("Skipped master-local WAL {}", lfs.getPath());
+      return;
+    }
     long timestamp = AbstractFSWALProvider.getTimestamp(lfs.getPath().getName());
     if (timestamp > 0) {
       // The name carries the WAL's creation time, which only bounds its entries from below. A WAL
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
index 5fbaf9e21e7..8a895ee20f4 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java
@@ -23,9 +23,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -99,6 +102,45 @@ public class TestWALInputFormat {
     assertEquals(9, lfss.size());
   }
 
+  @Test
+  public void testSkipsArchivedMasterLocalWALs() throws Exception {
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ",");
+    conf.setLong(WALInputFormat.START_TIME_KEY, 200);
+    conf.setLong(WALInputFormat.END_TIME_KEY, 300);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path inputDir = TEST_UTIL.getDataTestDirOnTestFS("master-local-wals");
+    Path nonEmpty = new Path(inputDir, "master.250$masterlocalwal$");
+    Path empty = new Path(inputDir, "nested/master.251$masterlocalwal$");
+    List<String> expected =
+      Arrays.asList("0000000000000016310", "server.250", "server.250.meta", "server.250.syncrep");
+    try {
+      for (String name : expected) {
+        try (FSDataOutputStream out = fs.create(new Path(inputDir, "nested/" + name))) {
+          out.write(1);
+        }
+      }
+      try (FSDataOutputStream out = fs.create(nonEmpty)) {
+        out.write(1);
+      }
+      fs.create(empty).close();
+
+      for (boolean ignoreEmptyFiles : new boolean[] { false, true }) {
+        conf.setBoolean(WALPlayer.IGNORE_EMPTY_FILES, ignoreEmptyFiles);
+        conf.set(FileInputFormat.INPUT_DIR, inputDir.toString());
+        List<String> actual = new WALInputFormat().getSplits(Job.getInstance(conf)).stream()
+          .map(split -> new Path(((WALInputFormat.WALSplit) split).getLogFileName()).getName())
+          .sorted().collect(Collectors.toList());
+        assertEquals(expected, actual);
+
+        conf.set(FileInputFormat.INPUT_DIR, nonEmpty + "," + empty);
+        assertTrue(new WALInputFormat().getSplits(Job.getInstance(conf)).isEmpty());
+      }
+    } finally {
+      fs.delete(inputDir, true);
+    }
+  }
+
   private static boolean isKept(FileSystem fs, long created, long mtime, long start, long end) {
     return isKept(fs, created, mtime, mtime, start, end);
   }