You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by pa...@apache.org on 2021/08/26 18:43:24 UTC
[hbase] branch branch-2.3 updated: HBASE-26223 Splitted WAL files
are deleted rather than archiving (#3627)
This is an automated email from the ASF dual-hosted git repository.
pankajkumar pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 354a3bf HBASE-26223 Splitted WAL files are deleted rather than archiving (#3627)
354a3bf is described below
commit 354a3bf92052829c2e7e89093ce7787cb8fd0d21
Author: Pankaj <pa...@apache.org>
AuthorDate: Fri Aug 27 00:12:56 2021 +0530
HBASE-26223 Splitted WAL files are deleted rather than archiving (#3627)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../org/apache/hadoop/hbase/master/SplitWALManager.java | 11 +++++++++--
.../hbase/master/procedure/SplitWALRemoteProcedure.java | 2 +-
.../java/org/apache/hadoop/hbase/wal/WALSplitUtil.java | 14 ++++++++++++++
.../apache/hadoop/hbase/master/TestSplitWALManager.java | 10 ++++++++++
4 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
index c08c80f..47f3e82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
import org.apache.hadoop.hbase.master.procedure.SplitWALProcedure;
@@ -41,6 +42,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.wal.WALSplitUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,6 +80,7 @@ public class SplitWALManager {
private final Path rootDir;
private final FileSystem fs;
private final Configuration conf;
+ private final Path walArchiveDir;
public SplitWALManager(MasterServices master) {
this.master = master;
@@ -86,6 +89,7 @@ public class SplitWALManager {
conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER));
this.rootDir = master.getMasterFileSystem().getWALRootDir();
this.fs = master.getMasterFileSystem().getWALFileSystem();
+ this.walArchiveDir = new Path(this.rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
}
public List<Procedure> splitWALs(ServerName crashedServer, boolean splitMeta)
@@ -116,8 +120,11 @@ public class SplitWALManager {
return logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
}
- public void deleteSplitWAL(String wal) throws IOException {
- fs.delete(new Path(wal), false);
+ /**
+ * Archive processed WAL
+ */
+ public void archive(String wal) throws IOException {
+ WALSplitUtil.moveWAL(this.fs, new Path(wal), this.walArchiveDir);
}
public void deleteWALDir(ServerName serverName) throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
index c829e51..54607e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
@@ -96,7 +96,7 @@ public class SplitWALRemoteProcedure extends ServerRemoteProcedure
protected void complete(MasterProcedureEnv env, Throwable error) {
if (error == null) {
try {
- env.getMasterServices().getSplitWALManager().deleteSplitWAL(walPath);
+ env.getMasterServices().getSplitWALManager().archive(walPath);
} catch (IOException e) {
LOG.warn("Failed split of {}; ignore...", walPath, e);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
index 2b38494..9567b5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
@@ -160,6 +160,20 @@ public final class WALSplitUtil {
}
/**
+ * Move WAL. Used to move processed WALs to archive or bad WALs to corrupt WAL dir.
+ * WAL may have already been moved; makes allowance.
+ */
+ public static void moveWAL(FileSystem fs, Path p, Path targetDir) throws IOException {
+ if (fs.exists(p)) {
+ if (!CommonFSUtils.renameAndSetModifyTime(fs, p, targetDir)) {
+ LOG.warn("Failed move of {} to {}", p, targetDir);
+ } else {
+ LOG.info("Moved {} to {}", p, targetDir);
+ }
+ }
+ }
+
+ /**
* Path to a file under RECOVERED_EDITS_DIR directory of the region found in <code>logEntry</code>
* named for the sequenceid in the passed <code>logEntry</code>: e.g.
* /hbase/some_table/2323432434/recovered.edits/2332. This method also ensures existence of
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java
index e1f3188..5cabe0e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -233,11 +234,20 @@ public class TestSplitWALManager {
ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0));
Assert.assertEquals(0, splitWALManager.getWALsToSplit(testServer, false).size());
+ // Validate the old WAL file archive dir
+ Path walRootDir = hmaster.getMasterFileSystem().getWALRootDir();
+ Path walArchivePath = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ FileSystem walFS = hmaster.getMasterFileSystem().getWALFileSystem();
+ int archiveFileCount = walFS.listStatus(walArchivePath).length;
+
procedures = splitWALManager.splitWALs(metaServer, true);
Assert.assertEquals(1, procedures.size());
ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0));
Assert.assertEquals(0, splitWALManager.getWALsToSplit(metaServer, true).size());
Assert.assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size());
+ // There should be archiveFileCount + 1 WALs after SplitWALProcedure finish
+ Assert.assertEquals("Splitted WAL files should be archived", archiveFileCount + 1,
+ walFS.listStatus(walArchivePath).length);
}
@Test