You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/04/03 21:39:56 UTC
svn commit: r1584346 - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-common/src/main/resources/
hbase-server/src/main/java/org/apache/hadoop/hbase/master/ hbase-...
Author: stack
Date: Thu Apr 3 19:39:55 2014
New Revision: 1584346
URL: http://svn.apache.org/r1584346
Log:
HBASE-10856 Enable distributed log replay as default
Modified:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java Thu Apr 3 19:39:55 2014
@@ -505,7 +505,7 @@ public class RecoverableZooKeeper {
Bytes.toStringBinary(data));
throw e;
}
- LOG.info("Node " + path + " already exists");
+ LOG.debug("Node " + path + " already exists");
throw e;
case CONNECTIONLOSS:
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Apr 3 19:39:55 2014
@@ -794,7 +794,10 @@ public final class HConstants {
/** Conf key that enables unflushed WAL edits directly being replayed to region servers */
public static final String DISTRIBUTED_LOG_REPLAY_KEY = "hbase.master.distributed.log.replay";
- public static final boolean DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG = false;
+ /**
+ * Default 'distributed log replay' as true since hbase 0.99.0
+ */
+ public static final boolean DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG = true;
public static final String DISALLOW_WRITES_IN_RECOVERING =
"hbase.regionserver.disallow.writes.when.recovering";
public static final boolean DEFAULT_DISALLOW_WRITES_IN_RECOVERING_CONFIG = false;
Modified: hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/resources/hbase-default.xml?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/hbase-common/src/main/resources/hbase-default.xml Thu Apr 3 19:39:55 2014
@@ -220,6 +220,17 @@ possible configurations would overwhelm
<description>The HLog file writer implementation.</description>
</property>
<property>
+ <name>hbase.master.distributed.log.replay</name>
+ <value>true</value>
+ <description>Enable 'distributed log replay' as default engine splitting
+ WAL files on server crash. This default is new in hbase 1.0. To fall
+ back to the old mode 'distributed log splitter', set the value to
+ 'false'. 'Disributed log replay' improves MTTR because it does not
+ write intermediate files. 'DLR' required that 'hfile.format.version'
+ be set to version 3 or higher.
+ </description>
+ </property>
+ <property>
<name>hbase.regionserver.global.memstore.size</name>
<value>0.4</value>
<description>Maximum size of all memstores in a region server before new
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java Thu Apr 3 19:39:55 2014
@@ -115,7 +115,7 @@ public class DeadServer {
}
public synchronized void finish(ServerName sn) {
- LOG.info("Finished processing " + sn);
+ LOG.debug("Finished processing " + sn);
this.numProcessing--;
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Thu Apr 3 19:39:55 2014
@@ -628,7 +628,7 @@ public class SplitLogManager extends Zoo
try {
if (regions == null) {
// remove all children under /home/recovering-regions
- LOG.info("Garbage collecting all recovering regions.");
+ LOG.debug("Garbage collecting all recovering region znodes");
ZKUtil.deleteChildrenRecursively(watcher, watcher.recoveringRegionsZNode);
} else {
for (String curRegion : regions) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Thu Apr 3 19:39:55 2014
@@ -88,7 +88,7 @@ public class CreateTableHandler extends
int timeout = conf.getInt("hbase.client.catalog.timeout", 10000);
// Need hbase:meta availability to create a table
try {
- if(catalogTracker.waitForMeta(timeout) == null) {
+ if (catalogTracker.waitForMeta(timeout) == null) {
throw new NotAllMetaRegionsOnlineException();
}
} catch (InterruptedException e) {
@@ -114,7 +114,9 @@ public class CreateTableHandler extends
// Use enabling state to tell if there is already a request for the same
// table in progress. This will introduce a new zookeeper call. Given
// createTable isn't a frequent operation, that should be ok.
- //TODO: now that we have table locks, re-evaluate above
+ // TODO: now that we have table locks, re-evaluate above -- table locks are not enough.
+ // We could have cleared the hbase.rootdir and not zk. How can we detect this case?
+ // Having to clean zk AND hdfs is awkward.
try {
if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName)) {
throw new TableExistsException(tableName);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java Thu Apr 3 19:39:55 2014
@@ -83,6 +83,6 @@ public class LogReplayHandler extends Ev
this.deadServers.finish(serverName);
}
// logReplay is the last step of SSH so log a line to indicate that
- LOG.info("Finished processing of shutdown of " + serverName);
+ LOG.info("Finished processing shutdown of " + serverName);
}
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java Thu Apr 3 19:39:55 2014
@@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.client.Mu
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.master.SplitLogManager;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -1972,7 +1973,13 @@ public class HLogSplitter {
* @return true when distributed log replay is turned on
*/
public static boolean isDistributedLogReplay(Configuration conf) {
- return conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
+ boolean dlr = conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
+ int version = conf.getInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Distributed log replay=" + dlr + ", " + HFile.FORMAT_VERSION_KEY + "=" + version);
+ }
+ // For distributed log replay, hfile version must be 3 at least; we need tag support.
+ return dlr && (version >= 3);
}
}