You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:06:07 UTC
svn commit: r1181410 - in /hbase/branches/0.89: bin/rolling-restart.sh
bin/stop-hbase.sh src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Author: nspiegelberg
Date: Tue Oct 11 02:06:07 2011
New Revision: 1181410
URL: http://svn.apache.org/viewvc?rev=1181410&view=rev
Log:
rolling restart and misc fixes
Summary:
(1) Always sleep 2 mins before restarting masters during rolling restart.
(2) Backup master always stalls 2 mins, giving primary a fair chance.
Previously it was doing this only if the /hbase/master ZNODE did not exist.
Sometimes the ZNODE exists even after a restart if cluster hasn't been properly
shutdown. So, doing this ensures that in the common case the primary will indeed
be the primary.
(3) bin/stop-hbase.sh doesn't need to kill backup masters, because cluster stop
via the admin API itself triggers all region servers as well as backup masters
to be shutdown. This is what was previously causing this confusing error:
bin/stop-hbase.sh
stopping hbase.............: no master to stop
because kill -0 of pid 19014 failed with status 1
4) In rolling-restart.sh, as a marginal improvement, shutting down the backup
master first and then the primary master. This is to avoid the backup trying to
take over if we kill the primary first. [Ideally the order should be dynamically
determined, but deferring that for future.]
Test Plan:
Ran various full and rolling restart tests on dev cluster.
DiffCamp Revision: 173364
Reviewed By: kranganathan
CC: kranganathan
Revert Plan:
OK
Modified:
hbase/branches/0.89/bin/rolling-restart.sh
hbase/branches/0.89/bin/stop-hbase.sh
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Modified: hbase/branches/0.89/bin/rolling-restart.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/rolling-restart.sh?rev=1181410&r1=1181409&r2=1181410&view=diff
==============================================================================
--- hbase/branches/0.89/bin/rolling-restart.sh (original)
+++ hbase/branches/0.89/bin/rolling-restart.sh Tue Oct 11 02:06:07 2011
@@ -62,23 +62,25 @@ if [ "$distMode" == 'false' ]; then
"$bin"/hbase-daemon.sh restart master
else
# stop all masters before re-start to avoid races for master znode
- "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
+ # TODO: Ideally, we should find out who the actual primary is
+ # dynamically and kill everyone else first.
+ echo "Backup master(s):"
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
- --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
+ --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
+
+ echo "Primary master:"
+ "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
+
+ echo "Sleeping 120 seconds (for /hbase/master znode expiration)..."
- # make sure the master znode has been deleted before continuing
- zparent=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.parent`
- if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
- zmaster=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.master`
- if [ "$zmaster" == "null" ]; then zmaster="master"; fi
- zmaster=$zparent/$zmaster
- echo -n "Waiting for Master ZNode to expire"
- while bin/hbase zkcli stat $zmaster >/dev/null 2>&1; do
- echo -n "."
- sleep 1
+ for ((i = 5; i <= 120; i+=5))
+ do
+ sleep 5
+ echo -n $i "sec.."
done
- echo #force a newline
+ echo ""
+ echo "Starting master and backup master(s)..."
# all masters are down, now restart
"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
Modified: hbase/branches/0.89/bin/stop-hbase.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/stop-hbase.sh?rev=1181410&r1=1181409&r2=1181410&view=diff
==============================================================================
--- hbase/branches/0.89/bin/stop-hbase.sh (original)
+++ hbase/branches/0.89/bin/stop-hbase.sh Tue Oct 11 02:06:07 2011
@@ -60,10 +60,5 @@ done
distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
if [ "$distMode" == 'true' ]
then
- # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message
- # stop any backup masters
- "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
- --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
-
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
fi
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1181410&r1=1181409&r2=1181410&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Oct 11 02:06:07 2011
@@ -230,22 +230,24 @@ public class HMaster extends Thread impl
zooKeeperWrapper.registerListener(zkMasterAddressWatcher);
// if we're a backup master, stall until a primary to writes his address
- if(conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
- LOG.debug("HMaster started in backup mode. " +
- "Stalling until master znode is written.");
- // this will only be a minute or so while the cluster starts up,
- // so don't worry about setting watches on the parent znode
- while (!zooKeeperWrapper.masterAddressExists()) {
- try {
- LOG.debug("Waiting for master address ZNode to be written " +
- "(Also watching cluster state node)");
- Thread.sleep(conf.getInt("zookeeper.session.timeout", 60 * 1000));
- } catch (InterruptedException e) {
- // interrupted = user wants to kill us. Don't continue
- throw new IOException("Interrupted waiting for master address");
- }
+ if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
+
+ // ephemeral node expiry will be detected between about 40 to 60 seconds;
+ // plus add a little extra since only ZK leader can expire nodes, and
+ // leader maybe a little bit delayed in getting info about the pings.
+ // Conservatively, just double the time.
+ int stallTime = conf.getInt("zookeeper.session.timeout", 60 * 1000) * 2;
+
+ LOG.debug("HMaster started in backup mode. Stall " + stallTime +
+ "ms giving primary master a fair chance to be the master...");
+ try {
+ Thread.sleep(stallTime);
+ } catch (InterruptedException e) {
+ // interrupted = user wants to kill us. Don't continue
+ throw new IOException("Interrupted waiting for master address");
}
}
+
this.zkMasterAddressWatcher.writeAddressToZooKeeper(this.address, true);
this.regionServerOperationQueue =
new RegionServerOperationQueue(this.conf, this.closed);