You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2018/08/01 19:33:39 UTC
hbase git commit: HBASE-20729: B & R BackupLogCleaner must ignore
ProcV2 WAL files
Repository: hbase
Updated Branches:
refs/heads/master 1d0fca370 -> d53a976e8
HBASE-20729: B & R BackupLogCleaner must ignore ProcV2 WAL files
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d53a976e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d53a976e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d53a976e
Branch: refs/heads/master
Commit: d53a976e8df2304da1c78d950cd1224402ced055
Parents: 1d0fca3
Author: Vladimir Rodionov <vr...@hortonworks.com>
Authored: Wed Aug 1 11:16:08 2018 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Aug 1 12:33:33 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/backup/impl/BackupSystemTable.java | 6 ++++++
.../hbase/backup/impl/IncrementalBackupManager.java | 10 +++++-----
.../hadoop/hbase/backup/master/BackupLogCleaner.java | 1 -
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d53a976e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java
index 1b36944..d177384 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.util.Bytes;
@@ -1116,6 +1117,11 @@ public final class BackupSystemTable implements Closeable {
List<FileStatus> fileStatuses = new ArrayList<>();
for (FileStatus file : files) {
+ String fn = file.getPath().getName();
+ if (fn.startsWith(WALProcedureStore.LOG_PREFIX)) {
+ ret.put(file, true);
+ continue;
+ }
String wal = file.getPath().toString();
Get get = createGetForCheckWALFile(wal);
getBuffer.add(get);
http://git-wip-us.apache.org/repos/asf/hbase/blob/d53a976e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
index a20f9b5..81652d8 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
@@ -101,7 +102,7 @@ public class IncrementalBackupManager extends BackupManager {
List<WALItem> logFromSystemTable =
getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo()
.getBackupRootDir());
- logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable);
+ logList = excludeAlreadyBackedUpAndProcV2WALs(logList, logFromSystemTable);
backupInfo.setIncrBackupFileList(logList);
return newTimestamps;
@@ -144,14 +145,13 @@ public class IncrementalBackupManager extends BackupManager {
getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo()
.getBackupRootDir());
- logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable);
+ logList = excludeAlreadyBackedUpAndProcV2WALs(logList, logFromSystemTable);
backupInfo.setIncrBackupFileList(logList);
return logList;
}
-
- private List<String> excludeAlreadyBackedUpWALs(List<String> logList,
+ private List<String> excludeAlreadyBackedUpAndProcV2WALs(List<String> logList,
List<WALItem> logFromSystemTable) {
Set<String> walFileNameSet = convertToSet(logFromSystemTable);
@@ -160,7 +160,7 @@ public class IncrementalBackupManager extends BackupManager {
Path p = new Path(logList.get(i));
String name = p.getName();
- if (walFileNameSet.contains(name)) {
+ if (walFileNameSet.contains(name) || name.startsWith(WALProcedureStore.LOG_PREFIX)) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d53a976e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
index 5ce11d1..8d9400f 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
@@ -97,7 +97,6 @@ public class BackupLogCleaner extends BaseLogCleanerDelegate {
LOG.warn("Backup system table is not available: {}", tnfe.getMessage());
return files;
}
-
List<FileStatus> list = new ArrayList<>();
Map<FileStatus, Boolean> walFilesDeletableMap = table.areWALFilesDeletable(files);
for (Map.Entry<FileStatus, Boolean> entry: walFilesDeletableMap.entrySet()) {