You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/06/08 22:09:12 UTC
hbase git commit: HBASE-15986 BackupServerUtil.getWALFilesOlderThan()
should handle oldWALs properly (Vladimir Rodionov)
Repository: hbase
Updated Branches:
refs/heads/HBASE-7912 658092788 -> 5901cbbe6
HBASE-15986 BackupServerUtil.getWALFilesOlderThan() should handle oldWALs properly (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5901cbbe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5901cbbe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5901cbbe
Branch: refs/heads/HBASE-7912
Commit: 5901cbbe6cc01dcb543c13c9609676e5c45a282e
Parents: 6580927
Author: tedyu <yu...@gmail.com>
Authored: Wed Jun 8 15:09:32 2016 -0700
Committer: Ted <yu...@gmail.com>
Committed: Wed Jun 8 15:09:32 2016 -0700
----------------------------------------------------------------------
.../hbase/backup/util/BackupClientUtil.java | 15 ++++++----
.../backup/impl/IncrementalBackupManager.java | 31 +++++++++++++-------
.../hbase/backup/util/BackupServerUtil.java | 24 ++++++++++-----
3 files changed, 46 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5901cbbe/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
index 422432d..2d47b72 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
@@ -128,11 +128,16 @@ public final class BackupClientUtil {
* @return host name
* @throws IOException exception
*/
- public static String parseHostFromOldLog(Path p) throws IOException {
- String n = p.getName();
- int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
- String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
- return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
+ public static String parseHostFromOldLog(Path p) {
+ try {
+ String n = p.getName();
+ int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
+ String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
+ return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
+ } catch (Exception e) {
+ LOG.warn("Skip log file (can't parse): " + p);
+ return null;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/5901cbbe/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
index 8e192f1..bd496ce 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -161,21 +161,24 @@ public class IncrementalBackupManager {
* @throws IOException
*/
private List<WALItem> getLogFilesFromBackupSystem(HashMap<String, Long> olderTimestamps,
- HashMap<String, Long> newestTimestamps, String backupRoot) throws IOException {
+ HashMap<String, Long> newestTimestamps, String backupRoot) throws IOException {
List<WALItem> logFiles = new ArrayList<WALItem>();
Iterator<WALItem> it = backupManager.getWALFilesFromBackupSystem();
while (it.hasNext()) {
WALItem item = it.next();
String rootDir = item.getBackupRoot();
- if(!rootDir.equals(backupRoot)) {
+ if (!rootDir.equals(backupRoot)) {
continue;
}
- String walFileName = item.getWalFile();
+ String walFileName = item.getWalFile();
String server = BackupServerUtil.parseHostNameFromLogFile(new Path(walFileName));
+ if (server == null) {
+ continue;
+ }
Long tss = getTimestamp(walFileName);
Long oldTss = olderTimestamps.get(server);
Long newTss = newestTimestamps.get(server);
- if (oldTss == null){
+ if (oldTss == null) {
logFiles.add(item);
continue;
}
@@ -205,10 +208,10 @@ public class IncrementalBackupManager {
* @throws IOException exception
*/
private List<String> getLogFilesForNewBackup(HashMap<String, Long> olderTimestamps,
- HashMap<String, Long> newestTimestamps, Configuration conf, String savedStartCode)
- throws IOException {
+ HashMap<String, Long> newestTimestamps, Configuration conf, String savedStartCode)
+ throws IOException {
LOG.debug("In getLogFilesForNewBackup()\n" + "olderTimestamps: " + olderTimestamps
- + "\n newestTimestamps: " + newestTimestamps);
+ + "\n newestTimestamps: " + newestTimestamps);
Path rootdir = FSUtils.getRootDir(conf);
Path logDir = new Path(rootdir, HConstants.HREGION_LOGDIR_NAME);
Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
@@ -239,6 +242,9 @@ public class IncrementalBackupManager {
for (FileStatus rs : rss) {
p = rs.getPath();
host = BackupServerUtil.parseHostNameFromLogFile(p);
+ if (host == null) {
+ continue;
+ }
FileStatus[] logs;
oldTimeStamp = olderTimestamps.get(host);
// It is possible that there is no old timestamp in hbase:backup for this host if
@@ -252,7 +258,7 @@ public class IncrementalBackupManager {
for (FileStatus log : logs) {
LOG.debug("currentLogFile: " + log.getPath().toString());
if (DefaultWALProvider.isMetaFile(log.getPath())) {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("Skip hbase:meta log file: " + log.getPath().getName());
}
continue;
@@ -274,12 +280,15 @@ public class IncrementalBackupManager {
p = oldlog.getPath();
currentLogFile = p.toString();
if (DefaultWALProvider.isMetaFile(p)) {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("Skip .meta log file: " + currentLogFile);
}
continue;
}
host = BackupClientUtil.parseHostFromOldLog(p);
+ if (host == null) {
+ continue;
+ }
currentLogTS = BackupClientUtil.getCreationTime(p);
oldTimeStamp = olderTimestamps.get(host);
/*
@@ -327,7 +336,7 @@ public class IncrementalBackupManager {
public boolean accept(Path path) {
// skip meta table log -- ts.meta file
if (DefaultWALProvider.isMetaFile(path)) {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("Skip .meta log file: " + path.getName());
}
return false;
@@ -336,7 +345,7 @@ public class IncrementalBackupManager {
try {
timestamp = BackupClientUtil.getCreationTime(path);
return timestamp > Long.valueOf(lastBackupTS);
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.warn("Cannot read timestamp of log file " + path);
return false;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5901cbbe/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
index 6389386..e008836 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
@@ -221,12 +221,17 @@ public final class BackupServerUtil {
* @return host name: port
* @throws IOException
*/
- public static String parseHostNameFromLogFile(Path p) throws IOException {
- if (isArchivedLogFile(p)) {
- return BackupClientUtil.parseHostFromOldLog(p);
- } else {
- ServerName sname = DefaultWALProvider.getServerNameFromWALDirectoryName(p);
- return sname.getHostname() + ":" + sname.getPort();
+ public static String parseHostNameFromLogFile(Path p) {
+ try {
+ if (isArchivedLogFile(p)) {
+ return BackupClientUtil.parseHostFromOldLog(p);
+ } else {
+ ServerName sname = DefaultWALProvider.getServerNameFromWALDirectoryName(p);
+ return sname.getHostname() + ":" + sname.getPort();
+ }
+ } catch (Exception e) {
+ LOG.warn("Skip log file (can't parse): " + p);
+ return null;
}
}
@@ -359,11 +364,14 @@ public final class BackupServerUtil {
return false;
}
String host = parseHostNameFromLogFile(p);
+ if(host == null) {
+ return false;
+ }
Long oldTimestamp = hostTimestampMap.get(host);
Long currentLogTS = BackupClientUtil.getCreationTime(p);
return currentLogTS <= oldTimestamp;
- } catch (IOException e) {
- LOG.error(e);
+ } catch (Exception e) {
+ LOG.warn("Can not parse"+ p, e);
return false;
}
}