You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/11/17 02:43:31 UTC
svn commit: r1202976 - in
/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase:
master/HMaster.java regionserver/wal/HLog.java
regionserver/wal/HLogSplitter.java
Author: nspiegelberg
Date: Thu Nov 17 01:43:31 2011
New Revision: 1202976
URL: http://svn.apache.org/viewvc?rev=1202976&view=rev
Log:
process splitLog() errors
Summary:
1. At startup the splitLog() errors are only ignored if
hbase.hlog.split.skip.errors property is set to true (default = true). This is a
pre-existing property which is set to false in titan configs.
2. In ProcessServerShutdown the splitLog() exceptions are not ignored.
(splitLog() itself might not throw an exception on a corrupted log file if
hbase.hlog.split.skip.errors is set to true). If splitLog() fails, then the
region-operation is requeued.
Test Plan:
most of the unit tests have so far passed. still running.
==
namenode kill test
killed namenode
all the regionservers died
as expected, the first rs znode expiry killed the master as well. Verified from
master logs that on znode expiry following happens
- attempt to splitlogs that fail
- requeue region op
- check fs state that forces master to abort
then restarted namenode and HBase. all the logs got properly processed. No
loadtest errors, only saw failuers.
==
create some log files and kill the hbase processes.
to force a log-splitting failure touchz a file in the .logs directory with the
name of an existing log-directory appended with -splitting. The first step where
the log-directory is renamed to *-splitting will fail.
saw that the master didn't complete the startup sequence. Got stuck in
splitLogAtStatrtup. No region assignments happned.
removed the offending file from the .logs directory and the master proceeded to
split the logs and assign regions.
Reviewers: kannan, liyintang
Reviewed By: kannan
CC: hbase-eng@lists, kannan, pkhemani
Differential Revision: 350815
Task ID: 772146
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1202976&r1=1202975&r2=1202976&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Nov 17 01:43:31 2011
@@ -892,39 +892,55 @@ public class HMaster extends Thread impl
* ad active region server.
*/
private void splitLogAfterStartup() {
+ boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
+ HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+ List<String> serverNames = new ArrayList<String>();
try {
- Path logsDirPath =
- new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
- try {
- if (!this.fs.exists(logsDirPath)) return;
- } catch (IOException e) {
- throw new RuntimeException("Could exists for " + logsDirPath, e);
- }
- FileStatus[] logFolders;
- try {
- logFolders = this.fs.listStatus(logsDirPath);
- } catch (IOException e) {
- throw new RuntimeException("Failed listing " + logsDirPath.toString(), e);
- }
- if (logFolders == null || logFolders.length == 0) {
- LOG.debug("No log files to split, proceeding...");
- return;
- }
- List<String> serverNames = new ArrayList<String>();
- for (FileStatus status : logFolders) {
- Path logDir = status.getPath();
- String serverName = logDir.getName();
- LOG.info("Found log folder : " + serverName);
- if (this.serverManager.getServerInfo(serverName) == null) {
- LOG.info("Log folder " + status.getPath() + " doesn't belong " +
- "to a known region server, splitting");
- serverNames.add(serverName);
- } else {
- LOG.info("Log folder " + status.getPath() +
- " belongs to an existing region server");
+ do {
+ try {
+ Path logsDirPath =
+ new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+ if (!this.fs.exists(logsDirPath)) return;
+ FileStatus[] logFolders = this.fs.listStatus(logsDirPath);
+ if (logFolders == null || logFolders.length == 0) {
+ LOG.debug("No log files to split, proceeding...");
+ return;
+ }
+ for (FileStatus status : logFolders) {
+ Path logDir = status.getPath();
+ String serverName = logDir.getName();
+ LOG.info("Found log folder : " + serverName);
+ if (this.serverManager.getServerInfo(serverName) == null) {
+ LOG.info("Log folder " + status.getPath() + " doesn't belong " +
+ "to a known region server, splitting");
+ serverNames.add(serverName);
+ } else {
+ LOG.info("Log folder " + status.getPath() +
+ " belongs to an existing region server");
+ }
+ }
+
+ splitLog(serverNames);
+ retrySplitting = false;
+ } catch (IOException ioe) {
+ LOG.warn("Failed splitting of " + serverNames, ioe);
+ // reset serverNames
+ serverNames = new ArrayList<String>();
+ if (!checkFileSystem()) {
+ LOG.warn("Bad Filesystem, exiting");
+ Runtime.getRuntime().halt(1);
+ }
+ try {
+ if (retrySplitting) {
+ Thread.sleep(30000); //30s
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted, returning w/o splitting at startup");
+ Thread.currentThread().interrupt();
+ retrySplitting = false;
+ }
}
- }
- splitLog(serverNames);
+ } while (retrySplitting);
} finally {
isSplitLogAfterStartupDone.set(true);
}
@@ -934,13 +950,13 @@ public class HMaster extends Thread impl
return (isSplitLogAfterStartupDone.get());
}
- public void splitLog(final String serverName) {
+ public void splitLog(final String serverName) throws IOException {
List<String> serverNames = new ArrayList<String>();
serverNames.add(serverName);
splitLog(serverNames);
}
- public void splitLog(final List<String> serverNames) {
+ public void splitLog(final List<String> serverNames) throws IOException {
long splitTime = 0, splitLogSize = 0, splitCount = 0;
List<String> realServerNames = new ArrayList<String>();
List<Path> logDirs = new ArrayList<Path>();
@@ -952,16 +968,9 @@ public class HMaster extends Thread impl
realServerNames.add(serverName);
Path splitDir = new Path(logDir.getParent(), logDir.getName()
+ HConstants.HLOG_SPLITTING_EXT);
- try {
- if (!this.fs.rename(logDir, splitDir)) {
- LOG.error("Failed log splitting because " +
+ if (!this.fs.rename(logDir, splitDir)) {
+ throw new IOException("Failed log splitting because " +
" failed fs.rename of " + logDir);
- return;
- }
- } catch (IOException ioe) {
- LOG.error("Failed log splitting because" +
- " failed fs.rename of " + logDir, ioe);
- return;
}
logDir = splitDir;
LOG.debug("Renamed region directory: " + splitDir);
@@ -971,15 +980,9 @@ public class HMaster extends Thread impl
}
logDirs.add(logDir);
ContentSummary contentSummary;
- try {
- contentSummary = fs.getContentSummary(logDir);
- splitCount += contentSummary.getFileCount();
- splitLogSize += contentSummary.getSpaceConsumed();
- } catch (IOException e) {
- LOG.error("Failed log splitting because" +
- " failed to get file system content summary", e);
- return;
- }
+ contentSummary = fs.getContentSummary(logDir);
+ splitCount += contentSummary.getFileCount();
+ splitLogSize += contentSummary.getSpaceConsumed();
}
splitTime = EnvironmentEdgeManager.currentTimeMillis();
if (distributedLogSplitting) {
@@ -988,16 +991,11 @@ public class HMaster extends Thread impl
splitLogManager.handleDeadWorker(realServerName);
}
try {
- try {
- splitLogManager.splitLogDistributed(logDirs);
- } catch (OrphanHLogAfterSplitException e) {
- LOG.warn("Retrying distributed splitting for " + serverNames
- + "because of:", e);
- splitLogManager.splitLogDistributed(logDirs);
- }
- } catch (IOException e) {
- LOG.error("Failed distributed splitting " + serverNames, e);
- return;
+ splitLogManager.splitLogDistributed(logDirs);
+ } catch (OrphanHLogAfterSplitException e) {
+ LOG.warn("Retrying distributed splitting for " + serverNames
+ + "because of:", e);
+ splitLogManager.splitLogDistributed(logDirs);
}
} else {
// splitLogLock ensures that dead region servers' logs are processed
@@ -1007,9 +1005,6 @@ public class HMaster extends Thread impl
try {
HLog.splitLog(this.rootdir, logDir, oldLogDir, this.fs,
getConfiguration());
- } catch (IOException e) {
- LOG.error("Failed splitting " + logDir.toString(), e);
- return;
} finally {
this.splitLogLock.unlock();
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1202976&r1=1202975&r2=1202976&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Thu Nov 17 01:43:31 2011
@@ -129,6 +129,7 @@ public class HLog implements Syncable {
static final Log LOG = LogFactory.getLog(HLog.class);
public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
static final byte [] METAROW = Bytes.toBytes("METAROW");
+ public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
/*
* Name of directory that holds recovered edits written by the wal log
@@ -1429,7 +1430,8 @@ public class HLog implements Syncable {
//TODO make a note on the conf rename and update hbase-site.xml if needed
int logFilesPerStep = conf.getInt("hbase.hlog.split.batch.size", 3);
boolean skipErrors =
- conf.getBoolean("hbase.hlog.split.skip.errors", false);
+ conf.getBoolean("hbase.hlog.split.skip.errors",
+ SPLIT_SKIP_ERRORS_DEFAULT);
lastSplitSize = 0;
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1202976&r1=1202975&r2=1202976&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java Thu Nov 17 01:43:31 2011
@@ -183,7 +183,8 @@ public class HLogSplitter {
boolean progress_failed = false;
- boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", true);
+ boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors",
+ HLog.SPLIT_SKIP_ERRORS_DEFAULT);
int interval = conf.getInt("hbase.splitlog.report.interval.loglines", 1024);
// How often to send a progress report (default 1/2 master timeout)
int period = conf.getInt("hbase.splitlog.report.period",