You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/05/04 17:19:48 UTC
svn commit: r1334028 - in /hbase/trunk: bin/hbase-daemon.sh
src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Author: stack
Date: Fri May 4 15:19:47 2012
New Revision: 1334028
URL: http://svn.apache.org/viewvc?rev=1334028&view=rev
Log:
HBASE-5844 Delete the region servers znode after a regions server crash
Modified:
hbase/trunk/bin/hbase-daemon.sh
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Modified: hbase/trunk/bin/hbase-daemon.sh
URL: http://svn.apache.org/viewvc/hbase/trunk/bin/hbase-daemon.sh?rev=1334028&r1=1334027&r2=1334028&view=diff
==============================================================================
--- hbase/trunk/bin/hbase-daemon.sh (original)
+++ hbase/trunk/bin/hbase-daemon.sh Fri May 4 15:19:47 2012
@@ -71,6 +71,15 @@ hbase_rotate_log ()
fi
}
+cleanZNode() {
+ if [ -f $HBASE_ZNODE_FILE ]; then
+ #call ZK to delete the node
+ ZNODE=`cat $HBASE_ZNODE_FILE`
+ $bin/hbase zkcli delete $ZNODE > /dev/null 2>&1
+ rm $HBASE_ZNODE_FILE
+ fi
+}
+
wait_until_done ()
{
p=$1
@@ -121,6 +130,7 @@ logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.
loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc
loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}"
pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid
+export HBASE_ZNODE_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.znode
if [ "$HBASE_USE_GC_LOGFILE" = "true" ]; then
export HBASE_GC_OPTS=" -Xloggc:${loggc}"
@@ -131,6 +141,9 @@ if [ "$HBASE_NICENESS" = "" ]; then
export HBASE_NICENESS=0
fi
+thiscmd=$0
+args=$@
+
case $startStop in
(start)
@@ -141,18 +154,23 @@ case $startStop in
exit 1
fi
fi
+ nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args < /dev/null > /dev/null 2>&1 &
+ ;;
+ (internal_start)
hbase_rotate_log $logout
hbase_rotate_log $loggc
echo starting $command, logging to $logout
# Add to the command log file vital stats on our environment.
echo "`date` Starting $command on `hostname`" >> $loglog
echo "`ulimit -a`" >> $loglog 2>&1
- nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \
+ nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \
--config "${HBASE_CONF_DIR}" \
- $command "$@" $startStop > "$logout" 2>&1 < /dev/null &
+ $command "$@" start > "$logout" &
echo $! > $pid
sleep 1; head "$logout"
+ wait
+ cleanZNode
;;
(stop)
@@ -178,8 +196,6 @@ case $startStop in
;;
(restart)
- thiscmd=$0
- args=$@
# stop the command
$thiscmd --config "${HBASE_CONF_DIR}" stop $command $args &
wait_until_done $!
@@ -193,6 +209,7 @@ case $startStop in
wait_until_done $!
;;
+
(*)
echo $usage
exit 1
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1334028&r1=1334027&r2=1334028&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri May 4 15:19:47 2012
@@ -19,6 +19,9 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.lang.Thread.UncaughtExceptionHandler;
@@ -725,6 +728,9 @@ public class HRegionServer extends Regio
} catch (KeeperException e) {
LOG.warn("Failed deleting my ephemeral node", e);
}
+ // We may have failed to delete the znode at the previous step, but
+ // we delete the file anyway: a second attempt to delete the znode is likely to fail again.
+ deleteMyEphemeralNodeOnDisk();
this.zooKeeper.close();
LOG.info("stopping server " + this.serverNameFromMasterPOV +
"; zookeeper connection closed.");
@@ -841,7 +847,8 @@ public class HRegionServer extends Regio
// No regions in RIT, we could stop waiting now.
if (this.regionsInTransitionInRS.isEmpty()) {
if (!isOnlineRegionsEmpty()) {
- LOG.info("We were exiting though online regions are not empty, because some regions failed closing");
+ LOG.info("We were exiting though online regions are not empty," +
+ " because some regions failed closing");
}
break;
}
@@ -911,6 +918,9 @@ public class HRegionServer extends Regio
// Set our ephemeral znode up in zookeeper now we have a name.
createMyEphemeralNode();
+ // Save it in a file, this will allow to see if we crash
+ writeMyEphemeralNodeOnDisk();
+
// Master sent us hbase.rootdir to use. Should be fully qualified
// path with file system specification included. Set 'fs.defaultFS'
// to match the filesystem on hbase.rootdir else underlying hadoop hdfs
@@ -945,11 +955,48 @@ public class HRegionServer extends Regio
return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
}
+ private String getMyEphemeralNodeFileName() {
+ return System.getenv().get("HBASE_ZNODE_FILE");
+ }
+
private void createMyEphemeralNode() throws KeeperException {
ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(),
HConstants.EMPTY_BYTE_ARRAY);
}
+ private void writeMyEphemeralNodeOnDisk() throws IOException {
+ String fileName = getMyEphemeralNodeFileName();
+
+ if (fileName == null) {
+ LOG.warn("No filename given to save the znode used, it won't be saved " +
+ "(Environment variable HBASE_ZNODE_FILE is not set).");
+ return;
+ }
+
+ FileWriter fstream = new FileWriter(fileName);
+ BufferedWriter out = new BufferedWriter(fstream);
+ try {
+ out.write(getMyEphemeralNodePath() + "\n");
+ } finally {
+ try {
+ out.close();
+ } finally {
+ fstream.close();
+ }
+ }
+ }
+
+ private void deleteMyEphemeralNodeOnDisk(){
+ String fileName = getMyEphemeralNodeFileName();
+
+ if (fileName == null){
+ return;
+ }
+
+ File f = new File(fileName);
+ f.delete();
+ }
+
private void deleteMyEphemeralNode() throws KeeperException {
ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
}