You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jg...@apache.org on 2010/07/27 23:57:02 UTC
svn commit: r979884 [1/2] - in /hbase/branches/0.90_master_rewrite: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/executor/
src/main/java/org/apache/hadoop/hbase/master/ sr...
Author: jgray
Date: Tue Jul 27 21:57:00 2010
New Revision: 979884
URL: http://svn.apache.org/viewvc?rev=979884&view=rev
Log:
HBASE-2695 [ZK-Master-Final-v4] Rest of Master/ZK cleanup and refactor
Added:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerMonitor.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java
Modified:
hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
hbase/branches/0.90_master_rewrite/src/main/resources/hbase-webapps/master/master.jsp
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
Modified: hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt (original)
+++ hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt Tue Jul 27 21:57:00 2010
@@ -18,6 +18,7 @@ Branch 0.90.0 - Master Rewrite Branch
new ZK tool ActiveMasterManager for master-side (part of
master cleanup and refactor)
HBASE-2696 [part2-v2-FinishRS_FixClient] ZooKeeper cleanup and refactor
+ HBASE-2695 [ZK-Master-Final-v4] Rest of Master/ZK cleanup and refactor
NEW FEATURES
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java?rev=979884&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java Tue Jul 27 21:57:00 2010
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+/**
+ * Interface to support the aborting of a given server or client.
+ *
+ * <p>This is used primarily for ZooKeeper usage when we could get an unexpected
+ * and fatal exception, requiring an abort.
+ *
+ * <p>Implemented by the Master, RegionServer, and TableServers (client).
+ */
+public interface Abortable {
+ /**
+ * Abort the server or client.
+ */
+ public void abort();
+}
\ No newline at end of file
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/MiniZooKeeperCluster.java Tue Jul 27 21:57:00 2010
@@ -62,6 +62,10 @@ public class MiniZooKeeperCluster {
this.clientPort = clientPort;
}
+ public int getClientPort() {
+ return clientPort;
+ }
+
public void setTickTime(int tickTime) {
this.tickTime = tickTime;
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java Tue Jul 27 21:57:00 2010
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.zookeeper
* Set of functions that are exposed by any HBase process (implemented by the
* master, region server, and client).
*/
-public interface ServerController {
+public interface ServerController extends Abortable {
/**
* Return the address of the current server.
*/
@@ -41,9 +41,4 @@ public interface ServerController {
* Get the ZooKeeper instance for this server.
*/
public ZooKeeperWatcher getZooKeeper();
-
- /**
- * Stub method into ServerStatus to move forward with ZK cleanup.
- */
- public void abortServer();
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue Jul 27 21:57:00 2010
@@ -1587,7 +1587,7 @@ public class HConnectionManager {
// The only method that really makes no sense here is get address
@Override
- public void abortServer() {
+ public void abort() {
if(zooKeeper != null) {
zooKeeper.close();
zooKeeper = null;
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java Tue Jul 27 21:57:00 2010
@@ -26,25 +26,24 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.executor.HBaseExecutorService.HBaseExecutorServiceType;
-import org.apache.hadoop.hbase.master.ServerManager;
/**
- * Abstract base class for all HBase event handlers. Subclasses should
- * implement the process() method where the actual handling of the event
+ * Abstract base class for all HBase event handlers. Subclasses should
+ * implement the process() method where the actual handling of the event
* happens.
- *
- * HBaseEventType is a list of ALL events (which also corresponds to messages -
- * either internal to one component or between components). The event type
- * names specify the component from which the event originated, and the
+ *
+ * HBaseEventType is a list of ALL events (which also corresponds to messages -
+ * either internal to one component or between components). The event type
+ * names specify the component from which the event originated, and the
* component which is supposed to handle it.
- *
- * Listeners can listen to all the events by implementing the interface
- * HBaseEventHandlerListener, and by registering themselves as a listener. They
+ *
+ * Listeners can listen to all the events by implementing the interface
+ * HBaseEventHandlerListener, and by registering themselves as a listener. They
* will be called back before and after the process of every event.
- *
- * TODO: Rename HBaseEvent and HBaseEventType to EventHandler and EventType
- * after ZK refactor as it currently would clash with EventType from ZK and
+ *
+ * TODO: Rename HBaseEvent and HBaseEventType to EventHandler and EventType
+ * after ZK refactor as it currently would clash with EventType from ZK and
* make the code very confusing.
*/
public abstract class HBaseEventHandler implements Runnable
@@ -54,16 +53,16 @@ public abstract class HBaseEventHandler
protected HBaseEventType eventType = HBaseEventType.NONE;
// is this a region server or master?
protected boolean isRegionServer;
- // name of the server - this is needed for naming executors in case of tests
+ // name of the server - this is needed for naming executors in case of tests
// where region servers may be co-located.
protected String serverName;
// listeners that are called before and after an event is processed
- protected static List<HBaseEventHandlerListener> eventHandlerListeners =
- Collections.synchronizedList(new ArrayList<HBaseEventHandlerListener>());
+ protected static List<HBaseEventHandlerListener> eventHandlerListeners =
+ Collections.synchronizedList(new ArrayList<HBaseEventHandlerListener>());
/**
- * This interface provides hooks to listen to various events received by the
- * queue. A class implementing this can listen to the updates by calling
+ * This interface provides hooks to listen to various events received by the
+ * queue. A class implementing this can listen to the updates by calling
* registerListener and stop receiving updates by calling unregisterListener
*/
public interface HBaseEventHandlerListener {
@@ -83,29 +82,29 @@ public abstract class HBaseEventHandler
*/
public enum HBaseEventType {
NONE (-1),
- // Messages originating from RS (NOTE: there is NO direct communication from
+ // Messages originating from RS (NOTE: there is NO direct communication from
// RS to Master). These are a result of RS updates into ZK.
RS2ZK_REGION_CLOSING (1), // RS is in process of closing a region
RS2ZK_REGION_CLOSED (2), // RS has finished closing a region
RS2ZK_REGION_OPENING (3), // RS is in process of opening a region
RS2ZK_REGION_OPENED (4), // RS has finished opening a region
-
- // Updates from master to ZK. This is done by the master and there is
+
+ // Updates from master to ZK. This is done by the master and there is
// nothing to process by either Master or RS
M2ZK_REGION_OFFLINE (50); // Master adds this region as offline in ZK
-
+
private final byte value;
-
+
/**
- * Called by the HMaster. Returns a name of the executor service given an
- * event type. Every event type has en entry - if the event should not be
+ * Called by the HMaster. Returns a name of the executor service given an
+ * event type. Every event type has en entry - if the event should not be
* handled just add the NONE executor.
* @return name of the executor service
*/
public HBaseExecutorServiceType getMasterExecutorForEvent() {
HBaseExecutorServiceType executorServiceType = null;
switch(this) {
-
+
case RS2ZK_REGION_CLOSING:
case RS2ZK_REGION_CLOSED:
executorServiceType = HBaseExecutorServiceType.MASTER_CLOSEREGION;
@@ -115,31 +114,31 @@ public abstract class HBaseEventHandler
case RS2ZK_REGION_OPENED:
executorServiceType = HBaseExecutorServiceType.MASTER_OPENREGION;
break;
-
+
case M2ZK_REGION_OFFLINE:
executorServiceType = HBaseExecutorServiceType.NONE;
break;
-
+
default:
throw new RuntimeException("Unhandled event type in the master.");
}
-
+
return executorServiceType;
}
/**
- * Called by the RegionServer. Returns a name of the executor service given an
- * event type. Every event type has en entry - if the event should not be
+ * Called by the RegionServer. Returns a name of the executor service given an
+ * event type. Every event type has en entry - if the event should not be
* handled just return a null executor name.
* @return name of the event service
*/
public static String getRSExecutorForEvent(String serverName) {
throw new RuntimeException("Unsupported operation.");
}
-
+
/**
- * Start the executor service that handles the passed in event type. The
- * server that starts these event executor services wants to handle these
+ * Start the executor service that handles the passed in event type. The
+ * server that starts these event executor services wants to handle these
* event types.
*/
public void startMasterExecutorService(String serverName) {
@@ -151,17 +150,29 @@ public abstract class HBaseEventHandler
}
public static void startRSExecutorService() {
-
+
}
HBaseEventType(int intValue) {
this.value = (byte)intValue;
}
-
+
public byte getByteValue() {
return value;
}
+ @Override
+ public String toString() {
+ switch(this) {
+ case RS2ZK_REGION_CLOSED: return "CLOSED";
+ case RS2ZK_REGION_CLOSING: return "CLOSING";
+ case RS2ZK_REGION_OPENED: return "OPENED";
+ case RS2ZK_REGION_OPENING: return "OPENING";
+ case M2ZK_REGION_OFFLINE: return "OFFLINE";
+ default: return this.name();
+ }
+ }
+
public static HBaseEventType fromByte(byte value) {
switch(value) {
case -1: return HBaseEventType.NONE;
@@ -176,12 +187,12 @@ public abstract class HBaseEventHandler
}
}
}
-
+
/**
* Default base class constructor.
- *
- * TODO: isRegionServer and serverName will go away once we do the HMaster
- * refactor. We will end up passing a ServerStatus which should tell us both
+ *
+ * TODO: isRegionServer and serverName will go away once we do the HMaster
+ * refactor. We will end up passing a ServerStatus which should tell us both
* the name and if it is a RS or master.
*/
public HBaseEventHandler(boolean isRegionServer, String serverName, HBaseEventType eventType) {
@@ -189,17 +200,17 @@ public abstract class HBaseEventHandler
this.eventType = eventType;
this.serverName = serverName;
}
-
+
/**
- * This is a wrapper around process, used to update listeners before and after
- * events are processed.
+ * This is a wrapper around process, used to update listeners before and after
+ * events are processed.
*/
public void run() {
// fire all beforeProcess listeners
for(HBaseEventHandlerListener listener : eventHandlerListeners) {
listener.beforeProcess(this);
}
-
+
// call the main process function
try {
process();
@@ -209,32 +220,32 @@ public abstract class HBaseEventHandler
// fire all afterProcess listeners
for(HBaseEventHandlerListener listener : eventHandlerListeners) {
- LOG.debug("Firing " + listener.getClass().getName() +
+ LOG.debug("Firing " + listener.getClass().getName() +
".afterProcess event listener for event " + eventType);
listener.afterProcess(this);
}
}
-
+
/**
- * This method is the main processing loop to be implemented by the various
+ * This method is the main processing loop to be implemented by the various
* subclasses.
*/
public abstract void process();
-
+
/**
* Subscribe to updates before and after processing events
*/
public static void registerListener(HBaseEventHandlerListener listener) {
eventHandlerListeners.add(listener);
}
-
+
/**
* Stop receiving updates before and after processing events
*/
public static void unregisterListener(HBaseEventHandlerListener listener) {
eventHandlerListeners.remove(listener);
}
-
+
public boolean isRegionServer() {
return isRegionServer;
}
@@ -247,7 +258,7 @@ public abstract class HBaseEventHandler
// TODO: check for isRegionServer here
return eventType.getMasterExecutorForEvent();
}
-
+
/**
* Return the event type
* @return
@@ -267,9 +278,9 @@ public abstract class HBaseEventHandler
}
serviceType.getExecutor(serverName).submit(this);
}
-
+
/**
- * Executes this event object in the caller's thread. This is a synchronous
+ * Executes this event object in the caller's thread. This is a synchronous
* way of executing the event.
*/
public void execute() {
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java Tue Jul 27 21:57:00 2010
@@ -88,6 +88,7 @@ public class HBaseExecutorService
LOG.debug("Executor service " + toString() + " already running on " + serverName);
return;
}
+ LOG.debug("Starting executor service [" + name + "]");
HBaseExecutorService.startExecutorService(name);
}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java?rev=979884&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java Tue Jul 27 21:57:00 2010
@@ -0,0 +1,245 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.executor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HMsg;
+import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Data serialized into ZooKeeper for region transitions.
+ */
+public class RegionTransitionData implements Writable {
+ /**
+ * Type of transition event (offline, opening, opened, closing, closed).
+ * Required.
+ */
+ private HBaseEventType eventType;
+
+ /** Region being transitioned. Required. */
+ private String regionName;
+
+ /** Server event originated from. Optional. */
+ private String serverName;
+
+ /** Time the event was created. Required but automatically set. */
+ private long timeStamp;
+
+ /** Temporary. Holds payload used doing transitions via heartbeats. */
+ private HMsg hmsg; // to be removed shortly once we stop using heartbeats
+
+ /**
+ * Writable constructor. Do not use directly.
+ */
+ public RegionTransitionData() {}
+
+ /**
+ * Construct data for a new region transition event with the specified event
+ * type and region name.
+ *
+ * <p>Used when the server name is not known (the master is setting it). This
+ * happens during cluster startup or during failure scenarios. When
+ * processing a failed regionserver, the master assigns the regions from that
+ * server to other servers though the region was never 'closed'. During
+ * master failover, the new master may have regions stuck in transition
+ * without a destination so may have to set regions offline and generate a new
+ * assignment.
+ *
+ * <p>Since only the master uses this constructor, the type should always be
+ * {@link HBaseEventType#M2ZK_REGION_OFFLINE}.
+ *
+ * @param eventType type of event
+ * @param regionName name of region
+ */
+ public RegionTransitionData(HBaseEventType eventType, String regionName) {
+ this(eventType, regionName, null);
+ }
+
+ /**
+ * Construct data for a new region transition event with the specified event
+ * type, region name, and server name.
+ *
+ * <p>Used when the server name is known (a regionserver is setting it).
+ *
+ * <p>Valid types for this constructor are {@link HBaseEventType#RS2ZK_REGION_CLOSING},
+ * {@link HBaseEventType#RS2ZK_REGION_CLOSED}, {@link HBaseEventType#RS2ZK_REGION_OPENING},
+ * and {@link HBaseEventType#RS2ZK_REGION_OPENED}.
+ *
+ * @param eventType type of event
+ * @param regionName name of region
+ * @param serverName name of server setting data
+ */
+ public RegionTransitionData(HBaseEventType eventType, String regionName,
+ String serverName) {
+ this(eventType, regionName, serverName, null);
+ }
+
+ /**
+ * Construct data for a fully-specified, old-format region transition event
+ * which uses HMsg/heartbeats.
+ *
+ * TODO: Remove this constructor once we stop using heartbeats.
+ *
+ * @param eventType
+ * @param regionName
+ * @param serverName
+ * @param hmsg
+ */
+ public RegionTransitionData(HBaseEventType eventType, String regionName,
+ String serverName, HMsg hmsg) {
+ this.eventType = eventType;
+ this.timeStamp = System.currentTimeMillis();
+ this.regionName = regionName;
+ this.serverName = serverName;
+ this.hmsg = hmsg;
+ }
+
+ /**
+ * Gets the type of region transition event.
+ *
+ * <p>One of:
+ * <ul>
+ * <li>{@link HBaseEventType#M2ZK_REGION_OFFLINE}
+ * <li>{@link HBaseEventType#RS2ZK_REGION_CLOSING}
+ * <li>{@link HBaseEventType#RS2ZK_REGION_CLOSED}
+ * <li>{@link HBaseEventType#RS2ZK_REGION_OPENING}
+ * <li>{@link HBaseEventType#RS2ZK_REGION_OPENED}
+ * </ul>
+ * @return type of region transition event
+ */
+ public HBaseEventType getEventType() {
+ return eventType;
+ }
+
+ /**
+ * Gets the encoded name of the region being transitioned.
+ *
+ * <p>Region name is required so this never returns null.
+ * @return region name
+ */
+ public String getRegionName() {
+ return regionName;
+ }
+
+ /**
+ * Gets the server the event originated from. If null, this event originated
+ * from the master.
+ *
+ * @return server name of originating regionserver, or null if from master
+ */
+ public String getServerName() {
+ return serverName;
+ }
+
+ /**
+ * Gets the timestamp when this event was created.
+ *
+ * @return time event was created
+ */
+ public long getTimeStamp() {
+ return timeStamp;
+ }
+
+ /**
+ * Gets the {@link HMsg} payload of this region transition event.
+ * @return heartbeat payload
+ */
+ public HMsg getHmsg() {
+ return hmsg;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ // the event type byte
+ eventType = HBaseEventType.fromByte(in.readByte());
+ // the timestamp
+ timeStamp = in.readLong();
+ // the encoded name of the region being transitioned
+ regionName = in.readUTF();
+ // remaining fields are optional so prefixed with boolean
+ // the name of the regionserver sending the data
+ if(in.readBoolean()) {
+ serverName = in.readUTF();
+ }
+ // hmsg
+ if(in.readBoolean()) {
+ hmsg = new HMsg();
+ hmsg.readFields(in);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(eventType.getByteValue());
+ out.writeLong(System.currentTimeMillis());
+ out.writeUTF(regionName);
+ // remaining fields are optional so prefixed with boolean
+ out.writeBoolean(serverName != null);
+ if(serverName != null) {
+ out.writeUTF(serverName);
+ }
+ out.writeBoolean(hmsg != null);
+ if(hmsg != null) {
+ hmsg.write(out);
+ }
+ }
+
+ /**
+ * Get the bytes for this instance. Throws a {@link RuntimeException} if
+ * there is an error deserializing this instance because it represents a code
+ * bug.
+ * @return binary representation of this instance
+ */
+ public byte [] getBytes() {
+ try {
+ return Writables.getBytes(this);
+ } catch(IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get an instance from bytes. Throws a {@link RuntimeException} if
+ * there is an error serializing this instance from bytes because it
+ * represents a code bug.
+ * @param bytes binary representation of this instance
+ * @return instance of this class
+ */
+ public static RegionTransitionData fromBytes(byte [] bytes) {
+ try {
+ RegionTransitionData data = new RegionTransitionData();
+ Writables.getWritable(bytes, data);
+ return data;
+ } catch(IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "region=" + regionName + ",server=" + serverName + ",state=" +
+ eventType;
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionEventData.java Tue Jul 27 21:57:00 2010
@@ -1,92 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.executor;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.io.Writable;
-
-public class RegionTransitionEventData implements Writable {
- private HBaseEventType hbEvent;
- private String rsName;
- private long timeStamp;
- private HMsg hmsg;
-
- public RegionTransitionEventData() {
- }
-
- public RegionTransitionEventData(HBaseEventType hbEvent, String rsName) {
- this(hbEvent, rsName, null);
- }
-
- public RegionTransitionEventData(HBaseEventType hbEvent, String rsName, HMsg hmsg) {
- this.hbEvent = hbEvent;
- this.rsName = rsName;
- this.timeStamp = System.currentTimeMillis();
- this.hmsg = hmsg;
- }
-
- public HBaseEventType getHbEvent() {
- return hbEvent;
- }
-
- public String getRsName() {
- return rsName;
- }
-
- public long getTimeStamp() {
- return timeStamp;
- }
-
- public HMsg getHmsg() {
- return hmsg;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- // the event type byte
- hbEvent = HBaseEventType.fromByte(in.readByte());
- // the hostname of the RS sending the data
- rsName = in.readUTF();
- // the timestamp
- timeStamp = in.readLong();
- if(in.readBoolean()) {
- // deserialized the HMsg from ZK
- hmsg = new HMsg();
- hmsg.readFields(in);
- }
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeByte(hbEvent.getByteValue());
- out.writeUTF(rsName);
- out.writeLong(System.currentTimeMillis());
- out.writeBoolean((hmsg != null));
- if(hmsg != null) {
- hmsg.write(out);
- }
- }
-
-}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java Tue Jul 27 21:57:00 2010
@@ -102,7 +102,7 @@ public class ActiveMasterManager extends
}
} catch (KeeperException ke) {
LOG.fatal("Received an unexpected KeeperException, aborting", ke);
- status.abortServer();
+ status.abort();
}
}
@@ -126,7 +126,7 @@ public class ActiveMasterManager extends
}
} catch (KeeperException ke) {
LOG.fatal("Received an unexpected KeeperException, aborting", ke);
- status.abortServer();
+ status.abort();
return;
}
// There is another active master, this is not a cluster startup
@@ -151,4 +151,19 @@ public class ActiveMasterManager extends
blockUntilBecomingActiveMaster();
}
}
+
+ public void stop() {
+ try {
+ // If our address is in ZK, delete it on our way out
+ HServerAddress zkAddress =
+ ZKUtil.getDataAsAddress(watcher, watcher.masterAddressZNode);
+ // TODO: redo this to make it atomic (only added for tests)
+ if(zkAddress != null &&
+ zkAddress.equals(address)) {
+ ZKUtil.deleteNode(watcher, watcher.masterAddressZNode);
+ }
+ } catch (KeeperException e) {
+ watcher.error("Error deleting our own master address node", e);
+ }
+ }
}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=979884&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Tue Jul 27 21:57:00 2010
@@ -0,0 +1,235 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.master.handler.MasterCloseRegionHandler;
+import org.apache.hadoop.hbase.master.handler.MasterOpenRegionHandler;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Manages region assignment.
+ *
+ * <p>Monitors ZooKeeper for events related to regions in transition.
+ *
+ * <p>Handles existing regions in transition during master failover.
+ */
+public class AssignmentManager extends ZooKeeperListener {
+ private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
+
+ private MasterStatus status;
+
+ private ServerManager serverManager;
+
+ private RegionManager regionManager;
+
+ private String serverName;
+
+// TODO: Eventually RIT will move here?
+// private final Map<String,RegionState> regionsInTransition =
+// new TreeMap<String,RegionState>();
+
+ /**
+ * Constructs a new assignment manager.
+ *
+ * <p>This manager must be started with {@link #start()}.
+ *
+ * @param watcher zookeeper watcher
+ * @param status master status
+ */
+ public AssignmentManager(ZooKeeperWatcher watcher, MasterStatus status,
+ ServerManager serverManager, RegionManager regionManager) {
+ super(watcher);
+ this.status = status;
+ this.serverManager = serverManager;
+ this.regionManager = regionManager;
+ serverName = status.getHServerAddress().toString();
+ }
+
+ /**
+ * Starts the assignment manager.
+ *
+ * <p>This includes registering itself with ZooKeeper and handling
+ * the initial state of whatever unassigned nodes already exist.
+ * @throws KeeperException
+ */
+ public void start() throws KeeperException {
+ watcher.registerListener(this);
+ if(status.isClusterStartup()) {
+ processStartup();
+ } else {
+ processFailover();
+ }
+ }
+
+ public synchronized void processStartup() throws KeeperException {
+ ZKAssign.deleteAllNodes(watcher);
+ ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.assignmentZNode);
+ }
+
+ /**
+ * Handle failover.
+ * @throws KeeperException
+ */
+ public synchronized void processFailover() throws KeeperException {
+ List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+ watcher.assignmentZNode);
+ if(nodes.isEmpty()) {
+ LOG.info("No regions in transition in ZK, nothing to do for failover");
+ return;
+ }
+ LOG.info("Failed-over master needs to process " + nodes.size() +
+ " regions in transition");
+ for(String regionName : nodes) {
+ RegionTransitionData data = ZKAssign.getData(watcher, regionName);
+ switch(data.getEventType()) {
+ case M2ZK_REGION_OFFLINE:
+ // TODO: Generate new assignment and send OPEN RPC
+ break;
+ case RS2ZK_REGION_CLOSING:
+ // TODO: Only need to deal with timeouts.
+ break;
+ case RS2ZK_REGION_CLOSED:
+ // TODO: Generate new assignment and send OPEN RPC
+ break;
+ case RS2ZK_REGION_OPENING:
+ // TODO: Only need to deal with timeouts.
+ break;
+ case RS2ZK_REGION_OPENED:
+ // TODO: Delete the node from ZK. Region successfully opened but not
+ // acknowledged.
+ break;
+ }
+ }
+ }
+
+ private synchronized void handleRegion(RegionTransitionData data) {
+ switch(data.getEventType()) {
+ case RS2ZK_REGION_CLOSED:
+ new MasterCloseRegionHandler(data.getEventType(), serverManager,
+ serverName, data.getRegionName(), data.getBytes())
+ .submit();
+ break;
+ case RS2ZK_REGION_OPENED:
+ case RS2ZK_REGION_OPENING:
+ new MasterOpenRegionHandler(data.getEventType(), serverManager,
+ serverName, data.getRegionName(), data.getBytes())
+ .submit();
+ break;
+ }
+ }
+
+ // ZooKeeper events
+
+ /**
+ * New unassigned node has been created.
+ *
+ * <p>This happens when an RS begins the OPENING or CLOSING of a region by
+ * creating an unassigned node.
+ *
+ * <p>When this happens we must:
+ * <ol>
+ * <li>Watch the node for further events</li>
+ * <li>Read and handle the state in the node</li>
+ * </ol>
+ */
+ @Override
+ public synchronized void nodeCreated(String path) {
+ if(path.startsWith(watcher.assignmentZNode)) {
+ try {
+ RegionTransitionData data = ZKAssign.getData(watcher, path);
+ if(data == null) {
+ return;
+ }
+ handleRegion(data);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZK exception reading unassigned node data", e);
+ status.abort();
+ }
+ }
+ }
+
+ /**
+ * Existing unassigned node has had data changed.
+ *
+ * <p>This happens when an RS transitions from OFFLINE to OPENING, or between
+ * OPENING/OPENED and CLOSING/CLOSED.
+ *
+ * <p>When this happens we must:
+ * <ol>
+ * <li>Watch the node for further events</li>
+ * <li>Read and handle the state in the node</li>
+ * </ol>
+ */
+ @Override
+ public synchronized void nodeDataChanged(String path) {
+ if(path.startsWith(watcher.assignmentZNode)) {
+ try {
+ RegionTransitionData data = ZKAssign.getData(watcher, path);
+ if(data == null) {
+ return;
+ }
+ handleRegion(data);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZK exception reading unassigned node data", e);
+ status.abort();
+ }
+ }
+ }
+
+ /**
+ * New unassigned node has been created.
+ *
+ * <p>This happens when an RS begins the OPENING or CLOSING of a region by
+ * creating an unassigned node.
+ *
+ * <p>When this happens we must:
+ * <ol>
+ * <li>Watch the node for further children changed events</li>
+ * <li>Watch all new children for changed events</li>
+ * <li>Read all children and handle them</li>
+ * </ol>
+ */
+ @Override
+ public synchronized void nodeChildrenChanged(String path) {
+ if(path.equals(watcher.assignmentZNode)) {
+ try {
+ List<NodeAndData> newNodes = ZKUtil.watchAndGetNewChildren(watcher,
+ watcher.assignmentZNode);
+ for(NodeAndData newNode : newNodes) {
+ LOG.debug("Handling new unassigned node: " + newNode);
+ handleRegion(RegionTransitionData.fromBytes(newNode.getData()));
+ }
+ } catch(KeeperException e) {
+ LOG.error("Unexpected ZK exception reading unassigned children", e);
+ status.abort();
+ }
+ }
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Jul 27 21:57:00 2010
@@ -82,17 +82,18 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.hadoop.hbase.util.Sleeper;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
+import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.DNS;
-import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
import com.google.common.collect.Lists;
@@ -106,8 +107,8 @@ import com.google.common.collect.Lists;
* @see HMasterRegionInterface
* @see Watcher
*/
-public class HMaster extends Thread implements HMasterInterface,
- HMasterRegionInterface, Watcher, MasterStatus {
+public class HMaster extends Thread
+implements HMasterInterface, HMasterRegionInterface, MasterStatus {
// MASTER is name of the webapp and the attribute name used stuffing this
//instance into web context.
public static final String MASTER = "master";
@@ -132,6 +133,13 @@ public class HMaster extends Thread impl
private ZooKeeperWatcher zooKeeperWrapper;
// Manager and zk listener for master election
private ActiveMasterManager activeMasterManager;
+ // Cluster status zk tracker and local setter
+ private ClusterStatusTracker clusterStatusTracker;
+ // Root region location tracker
+ private RootRegionTracker rootRegionTracker;
+ // Region server tracker
+ private RegionServerTracker regionServerTracker;
+
// A Sleeper that sleeps for threadWakeFrequency; sleep if nothing todo.
private final Sleeper sleeper;
// RPC server for the HMaster
@@ -147,6 +155,9 @@ public class HMaster extends Thread impl
// region manager to deal with region specific stuff
private final RegionManager regionManager;
+ // manager of assignment nodes in zookeeper
+ private final AssignmentManager assignmentManager;
+
// True if this is the master that started the cluster.
boolean isClusterStartup;
@@ -167,7 +178,7 @@ public class HMaster extends Thread impl
* <li>Block until becoming active master
* </ol>
*/
- public HMaster(Configuration conf) throws IOException {
+ public HMaster(Configuration conf) throws IOException, KeeperException {
// initialize some variables
this.conf = conf;
// set the thread name
@@ -199,7 +210,8 @@ public class HMaster extends Thread impl
*/
zooKeeperWrapper =
new ZooKeeperWatcher(conf, getHServerAddress().toString(), this);
- isClusterStartup = (zooKeeperWrapper.scanRSDirectory().size() == 0);
+ isClusterStartup = 0 ==
+ ZKUtil.getNumberOfChildren(zooKeeperWrapper, zooKeeperWrapper.rsZNode);
/*
* 3. Initialize master components.
@@ -210,9 +222,15 @@ public class HMaster extends Thread impl
this.connection = ServerConnectionManager.getConnection(conf);
this.regionServerOperationQueue = new RegionServerOperationQueue(conf, closed);
this.metrics = new MasterMetrics(this.getName());
+ clusterStatusTracker = new ClusterStatusTracker(zooKeeperWrapper, this);
+ rootRegionTracker = new RootRegionTracker(zooKeeperWrapper, this);
fileSystemManager = new FileSystemManager(conf, this);
serverManager = new ServerManager(this, metrics, regionServerOperationQueue);
- regionManager = new RegionManager(this);
+ regionManager = new RegionManager(this, rootRegionTracker);
+ assignmentManager = new AssignmentManager(zooKeeperWrapper, this,
+ serverManager, regionManager);
+ regionServerTracker = new RegionServerTracker(zooKeeperWrapper, this,
+ serverManager);
// create a sleeper to sleep for a configured wait frequency
int threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
this.sleeper = new Sleeper(threadWakeFrequency, this.closed);
@@ -235,6 +253,7 @@ public class HMaster extends Thread impl
activeMasterManager.blockUntilBecomingActiveMaster();
// We are the active master now.
+ clusterStatusTracker.setClusterUp();
LOG.info("Server has become the active/primary master. Address is " +
this.address.toString());
@@ -323,6 +342,7 @@ public class HMaster extends Thread impl
}
this.rpcServer.stop();
this.regionManager.stop();
+ this.activeMasterManager.stop();
this.zooKeeperWrapper.close();
HBaseExecutorService.shutdown();
LOG.info("HMaster main thread exiting");
@@ -424,7 +444,7 @@ public class HMaster extends Thread impl
* Get the ZK wrapper object - needed by master_jsp.java
* @return the zookeeper wrapper
*/
- public ZooKeeperWrapper getZooKeeperWrapper() {
+ public ZooKeeperWatcher getZooKeeperWatcher() {
return this.zooKeeperWrapper;
}
/**
@@ -445,11 +465,21 @@ public class HMaster extends Thread impl
*/
private void joinCluster() {
LOG.debug("Checking cluster state...");
- HServerAddress rootLocation =
- this.zooKeeperWrapper.readRootRegionLocation();
- List<HServerAddress> addresses = this.zooKeeperWrapper.scanRSDirectory();
+ HServerAddress rootLocation = null;
+ List<HServerAddress> addresses = null;
+ try {
+ clusterStatusTracker.start();
+ rootRegionTracker.start();
+ regionServerTracker.start();
+ rootLocation = rootRegionTracker.getRootRegionLocation();
+ addresses = regionServerTracker.getOnlineServers();
+ } catch(KeeperException e) {
+ LOG.fatal("Unexpected ZK exception initializing trackers", e);
+ abort();
+ return;
+ }
// Check if this is a fresh start of the cluster
- if (addresses.isEmpty()) {
+ if (isClusterStartup) {
LOG.debug("Master fresh start, proceeding with normal startup");
fileSystemManager.splitLogAfterStartup();
return;
@@ -504,14 +534,20 @@ public class HMaster extends Thread impl
*/
private void startServiceThreads() {
try {
- // Start the unassigned watcher - which will create the unassigned region
- // in ZK. This is needed before RegionManager() constructor tries to assign
- // the root region.
- ZKUnassignedWatcher.start(this.conf, this);
// start the "close region" executor service
- HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(address.toString());
+ HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(
+ address.toString());
// start the "open region" executor service
HBaseEventType.RS2ZK_REGION_OPENED.startMasterExecutorService(address.toString());
+ // Start the assignment manager. Creates the unassigned node in ZK
+ // if it does not exist and handles regions in transition if a failed-over
+ // master. This is needed before RegionManager() constructor tries to
+ // assign the root region.
+ try {
+ assignmentManager.start();
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ }
// start the region manager
this.regionManager.start();
// Put up info server.
@@ -608,7 +644,11 @@ public class HMaster extends Thread impl
public void shutdown() {
LOG.info("Cluster shutdown requested. Starting to quiesce servers");
this.shutdownRequested.set(true);
- this.zooKeeperWrapper.setClusterState(false);
+ try {
+ clusterStatusTracker.setClusterDown();
+ } catch (KeeperException e) {
+ LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+ }
}
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
@@ -967,47 +1007,6 @@ public class HMaster extends Thread impl
return status;
}
- /**
- * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.WatchedEvent)
- */
- @Override
- public void process(WatchedEvent event) {
- LOG.debug("Event " + event.getType() +
- " with state " + event.getState() +
- " with path " + event.getPath());
- // Master should kill itself if its session expired or if its
- // znode was deleted manually (usually for testing purposes)
- if(event.getState() == KeeperState.Expired ||
- (event.getType().equals(EventType.NodeDeleted) &&
- event.getPath().equals(this.zooKeeperWrapper.getMasterElectionZNode())) &&
- !shutdownRequested.get()) {
-
- LOG.info("Master lost its znode, trying to get a new one");
-
- // Can we still be the master? If not, goodbye
-
- zooKeeperWrapper.close();
- try {
- // TODO: this is broken, we should just shutdown now not restart
- zooKeeperWrapper =
- new ZooKeeperWatcher(conf, HMaster.class.getName(), this);
- zooKeeperWrapper.registerListener(this);
- activeMasterManager = new ActiveMasterManager(zooKeeperWrapper,
- this.address, this);
- activeMasterManager.blockUntilBecomingActiveMaster();
-
- // we are a failed over master, reset the fact that we started the
- // cluster
- setClusterStartup(false);
- // Verify the cluster to see if anything happened while we were away
- joinCluster();
- } catch (Exception e) {
- LOG.error("Killing master because of", e);
- System.exit(1);
- }
- }
- }
-
private static void printUsageAndExit() {
System.err.println("Usage: Master [opts] start|stop");
System.err.println(" start Start Master. If local mode, start Master and RegionServer in same JVM");
@@ -1042,7 +1041,8 @@ public class HMaster extends Thread impl
static class LocalHMaster extends HMaster {
private MiniZooKeeperCluster zkcluster = null;
- public LocalHMaster(Configuration conf) throws IOException {
+ public LocalHMaster(Configuration conf)
+ throws IOException, KeeperException {
super(conf);
}
@@ -1175,7 +1175,7 @@ public class HMaster extends Thread impl
}
@Override
- public void abortServer() {
+ public void abort() {
this.startShutdown();
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java Tue Jul 27 21:57:00 2010
@@ -28,7 +28,8 @@ import org.apache.hadoop.hbase.HServerIn
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
/**
* ProcessRegionOpen is instantiated when a region server reports that it is
@@ -115,8 +116,13 @@ public class ProcessRegionOpen extends P
} else {
masterStatus.getRegionManager().removeRegion(regionInfo);
}
- masterStatus.getZooKeeper().deleteUnassignedRegion(
- regionInfo.getEncodedName());
+ try {
+ ZKAssign.deleteOpenedNode(masterStatus.getZooKeeper(),
+ regionInfo.getEncodedName());
+ } catch (KeeperException e) {
+ LOG.error("ZK error deleting opened node", e);
+ throw new IOException(e);
+ }
return true;
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Tue Jul 27 21:57:00 2010
@@ -49,7 +49,6 @@ import org.apache.hadoop.hbase.HServerIn
import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -57,7 +56,11 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
/**
* Class to manage assigning regions to servers, state of root and meta, etc.
@@ -65,6 +68,8 @@ import org.apache.hadoop.hbase.zookeeper
public class RegionManager {
protected static final Log LOG = LogFactory.getLog(RegionManager.class);
+ private static final boolean ABORT_ON_ZK_ERROR = false;
+
private AtomicReference<HServerAddress> rootRegionLocation =
new AtomicReference<HServerAddress>(null);
@@ -81,7 +86,7 @@ public class RegionManager {
private static final byte[] OVERLOADED = Bytes.toBytes("Overloaded");
private static final byte [] META_REGION_PREFIX = Bytes.toBytes(".META.,");
-
+
private static int threadWakeFrequency;
/**
@@ -99,15 +104,16 @@ public class RegionManager {
*/
final SortedMap<String, RegionState> regionsInTransition =
Collections.synchronizedSortedMap(new TreeMap<String, RegionState>());
-
- // regions in transition are also recorded in ZK using the zk wrapper
- final ZooKeeperWrapper zkWrapper;
+
+ // regions in transition are also recorded in ZK using the zk watcher
+ final ZooKeeperWatcher zooKeeper;
// How many regions to assign a server at a time.
private final int maxAssignInOneGo;
final MasterStatus masterStatus;
private final LoadBalancer loadBalancer;
+ final RootRegionTracker rootRegionTracker;
/** Set of regions to split. */
private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
@@ -132,15 +138,17 @@ public class RegionManager {
private final int zooKeeperNumRetries;
private final int zooKeeperPause;
- RegionManager(MasterStatus masterStatus) throws IOException {
+ RegionManager(MasterStatus masterStatus, RootRegionTracker rootRegionTracker)
+ throws IOException {
Configuration conf = masterStatus.getConfiguration();
this.masterStatus = masterStatus;
- threadWakeFrequency =
+ this.rootRegionTracker = rootRegionTracker;
+ threadWakeFrequency =
masterStatus.getConfiguration().getInt(
- HConstants.THREAD_WAKE_FREQUENCY,
+ HConstants.THREAD_WAKE_FREQUENCY,
HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
- this.zkWrapper = masterStatus.getZooKeeper();
+ this.zooKeeper = masterStatus.getZooKeeper();
this.maxAssignInOneGo = conf.getInt("hbase.regions.percheckin", 10);
this.loadBalancer = new LoadBalancer(conf);
@@ -177,13 +185,14 @@ public class RegionManager {
if (!masterStatus.getShutdownRequested().get()) {
synchronized (regionsInTransition) {
String regionName = HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString();
- byte[] data = null;
try {
- data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
- } catch (IOException e) {
- LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
+ ZKAssign.createNodeOffline(zooKeeper,
+ HRegionInfo.ROOT_REGIONINFO.getEncodedName(), HMaster.MASTER);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZK exception creating offline node when " +
+ "trying to reassign root region", e);
+ if(ABORT_ON_ZK_ERROR) masterStatus.abort();
}
- zkWrapper.createUnassignedRegion(HRegionInfo.ROOT_REGIONINFO.getEncodedName(), data);
LOG.debug("Created UNASSIGNED zNode " + regionName + " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
RegionState s = new RegionState(HRegionInfo.ROOT_REGIONINFO, RegionState.State.UNASSIGNED);
regionsInTransition.put(regionName, s);
@@ -244,8 +253,9 @@ public class RegionManager {
final ArrayList<HMsg> returnMsgs) {
boolean isMetaAssign = false;
for (RegionState s : regionsToAssign) {
- if (s.getRegionInfo().isMetaRegion())
+ if (s.getRegionInfo().isMetaRegion()) {
isMetaAssign = true;
+ }
}
int nRegionsToAssign = regionsToAssign.size();
int otherServersRegionsCount =
@@ -341,13 +351,14 @@ public class RegionManager {
LOG.info("Assigning region " + regionName + " to " + sinfo.getServerName());
rs.setPendingOpen(sinfo.getServerName());
synchronized (this.regionsInTransition) {
- byte[] data = null;
try {
- data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
- } catch (IOException e) {
- LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
+ ZKAssign.createNodeOffline(zooKeeper,
+ rs.getRegionInfo().getEncodedName(), HMaster.MASTER);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZK exception creating offline node when " +
+ "trying to create offline node for region", e);
+ if(ABORT_ON_ZK_ERROR) masterStatus.abort();
}
- zkWrapper.createUnassignedRegion(rs.getRegionInfo().getEncodedName(), data);
LOG.debug("Created UNASSIGNED zNode " + regionName + " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
this.regionsInTransition.put(regionName, rs);
}
@@ -565,7 +576,7 @@ public class RegionManager {
RegionDirFilter rdf = new RegionDirFilter();
for(FileStatus tabledir : tableDirs) {
if(tabledir.isDir()) {
- regionDirs =
+ regionDirs =
masterStatus.getFileSystemManager().getFileSystem().listStatus(
tabledir.getPath(), rdf);
regions += regionDirs.length;
@@ -637,12 +648,11 @@ public class RegionManager {
} catch(Exception iex) {
LOG.warn("meta scanner", iex);
}
- // TODO: Why did we getInstance again? We should have it local?
-// ZooKeeperWrapper zkw = ZooKeeperWrapper.getInstance(
-// masterStatus.getConfiguration(),
-// masterStatus.getHServerAddress().toString());
- zkWrapper.clearRSDirectory();
- zkWrapper.close();
+ try {
+ ZKUtil.deleteChildrenRecursively(zooKeeper, zooKeeper.rsZNode);
+ } catch (KeeperException e) {
+ LOG.error("Unable to delete RS nodes during shutdown", e);
+ }
}
/**
@@ -996,15 +1006,14 @@ public class RegionManager {
synchronized(this.regionsInTransition) {
s = regionsInTransition.get(info.getRegionNameAsString());
if (s == null) {
- byte[] data = null;
try {
- data = Writables.getBytes(new RegionTransitionEventData(HBaseEventType.M2ZK_REGION_OFFLINE, HMaster.MASTER));
- } catch (IOException e) {
- // TODO: Review what we should do here. If Writables work this
- // should never happen
- LOG.error("Error creating event data for " + HBaseEventType.M2ZK_REGION_OFFLINE, e);
+ ZKAssign.createNodeOffline(zooKeeper,
+ info.getEncodedName(), HMaster.MASTER);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZK exception creating offline node when " +
+ "trying to reassign root region", e);
+ if(ABORT_ON_ZK_ERROR) masterStatus.abort();
}
- zkWrapper.createUnassignedRegion(info.getEncodedName(), data);
LOG.debug("Created UNASSIGNED zNode " + info.getRegionNameAsString() + " in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
s = new RegionState(info, RegionState.State.UNASSIGNED);
regionsInTransition.put(info.getRegionNameAsString(), s);
@@ -1231,11 +1240,13 @@ public class RegionManager {
private void writeRootRegionLocationToZooKeeper(HServerAddress address) {
for (int attempt = 0; attempt < zooKeeperNumRetries; ++attempt) {
- if (zkWrapper.writeRootRegionLocation(address)) {
+ try {
+ rootRegionTracker.setRootRegionLocation(address);
return;
+ } catch (KeeperException e) {
+ LOG.info("ZK exception writing root region location", e);
+ sleep(attempt);
}
-
- sleep(attempt);
}
LOG.error("Failed to write root region location to ZooKeeper after " +
@@ -1252,7 +1263,13 @@ public class RegionManager {
writeRootRegionLocationToZooKeeper(address);
synchronized (rootRegionLocation) {
// the root region has been assigned, remove it from transition in ZK
- zkWrapper.deleteUnassignedRegion(HRegionInfo.ROOT_REGIONINFO.getEncodedName());
+ try {
+ ZKAssign.deleteOpenedNode(zooKeeper,
+ HRegionInfo.ROOT_REGIONINFO.getEncodedName());
+ } catch (KeeperException e) {
+ LOG.error("Exception deleting root region unassigned node", e);
+ if(ABORT_ON_ZK_ERROR) masterStatus.abort();
+ }
rootRegionLocation.set(new HServerAddress(address));
rootRegionLocation.notifyAll();
}
@@ -1388,7 +1405,9 @@ public class RegionManager {
LoadBalancer(Configuration conf) {
this.slop = conf.getFloat("hbase.regions.slop", (float)0.3);
- if (this.slop <= 0) this.slop = 1;
+ if (this.slop <= 0) {
+ this.slop = 1;
+ }
//maxRegToClose to constrain balance closing per one iteration
// -1 to turn off
// TODO: change default in HBASE-862, need a suggestion
@@ -1462,16 +1481,18 @@ public class RegionManager {
SortedMap<HServerLoad, Set<String>> loadToServers =
masterStatus.getServerManager().getLoadToServers();
// check if server most loaded
- if (!loadToServers.get(loadToServers.lastKey()).contains(srvName))
+ if (!loadToServers.get(loadToServers.lastKey()).contains(srvName)) {
return 0;
+ }
// this server is most loaded, we will try to unload it by lowest
// loaded servers
int avgLoadMinusSlop = (int)Math.floor(avgLoad * (1 - this.slop)) - 1;
int lowestLoad = loadToServers.firstKey().getNumberOfRegions();
- if(lowestLoad >= avgLoadMinusSlop)
+ if(lowestLoad >= avgLoadMinusSlop) {
return 0; // there is no low loaded servers
+ }
int lowSrvCount = loadToServers.get(loadToServers.firstKey()).size();
int numRegionsToClose = 0;
@@ -1496,7 +1517,9 @@ public class RegionManager {
NavigableMap<String, String> getRegionsInTransition() {
NavigableMap<String, String> result = new TreeMap<String, String>();
synchronized (this.regionsInTransition) {
- if (this.regionsInTransition.isEmpty()) return result;
+ if (this.regionsInTransition.isEmpty()) {
+ return result;
+ }
for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
result.put(e.getKey(), e.getValue().toString());
}
@@ -1511,7 +1534,9 @@ public class RegionManager {
boolean clearFromInTransition(final byte [] regionname) {
boolean result = false;
synchronized (this.regionsInTransition) {
- if (this.regionsInTransition.isEmpty()) return result;
+ if (this.regionsInTransition.isEmpty()) {
+ return result;
+ }
for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
if (Bytes.equals(regionname, e.getValue().getRegionName())) {
this.regionsInTransition.remove(e.getKey());
@@ -1695,7 +1720,7 @@ public class RegionManager {
return Bytes.compareTo(getRegionName(), o.getRegionName());
}
}
-
+
/*
* When we find rows in a meta region that has an empty HRegionInfo, we
* clean them up here.
@@ -1719,7 +1744,7 @@ public class RegionManager {
}
}
}
-
+
// TODO ryan rework this function
/*
* Get HRegionInfo from passed META map of row values.
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerMonitor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerMonitor.java?rev=979884&view=auto
==============================================================================
(empty)
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Jul 27 21:57:00 2010
@@ -243,10 +243,8 @@ public class ServerManager {
HServerLoad load = useInfoLoad ? info.getLoad() : new HServerLoad();
String serverName = info.getServerName();
info.setLoad(load);
- // We must set this watcher here because it can be set on a fresh start
- // or on a failover
- Watcher watcher = new ServerExpirer(new HServerInfo(info));
- masterStatus.getZooKeeper().updateRSLocationGetWatch(info, watcher);
+ // TODO: Why did we update the RS location ourself? Shouldn't RS do this?
+// masterStatus.getZooKeeper().updateRSLocationGetWatch(info, watcher);
this.serversToServerInfo.put(serverName, info);
this.serversToLoad.put(serverName, load);
synchronized (this.loadToServers) {
@@ -848,7 +846,7 @@ public class ServerManager {
* Expire the passed server. Add it to list of deadservers and queue a
* shutdown processing.
*/
- private synchronized void expireServer(final HServerInfo hsi) {
+ public synchronized void expireServer(final HServerInfo hsi) {
// First check a server to expire. ServerName is of the form:
// <hostname> , <port> , <startcode>
String serverName = hsi.getServerName();
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java Tue Jul 27 21:57:00 2010
@@ -1,184 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
-import org.apache.hadoop.hbase.master.handler.MasterCloseRegionHandler;
-import org.apache.hadoop.hbase.master.handler.MasterOpenRegionHandler;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper.ZNodePathAndData;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-
-/**
- * Watches the UNASSIGNED znode in ZK for the master, and handles all events
- * relating to region transitions.
- */
-public class ZKUnassignedWatcher implements Watcher {
- private static final Log LOG = LogFactory.getLog(ZKUnassignedWatcher.class);
-
- private ZooKeeperWrapper zkWrapper;
- String serverName;
- ServerManager serverManager;
-
- public static void start(Configuration conf, MasterStatus masterStatus)
- throws IOException {
- new ZKUnassignedWatcher(conf, masterStatus);
- LOG.debug("Started ZKUnassigned watcher");
- }
-
- public ZKUnassignedWatcher(Configuration conf, MasterStatus masterStatus)
- throws IOException {
- this.serverName = masterStatus.getHServerAddress().toString();
- this.serverManager = masterStatus.getServerManager();
- zkWrapper = masterStatus.getZooKeeper();
- String unassignedZNode = zkWrapper.getRegionInTransitionZNode();
-
- // If the UNASSIGNED ZNode exists and this is a fresh cluster start, then
- // delete it.
- if(masterStatus.isClusterStartup() && zkWrapper.exists(unassignedZNode, false)) {
- LOG.info("Cluster start, but found " + unassignedZNode + ", deleting it.");
- try {
- zkWrapper.deleteZNode(unassignedZNode, true);
- } catch (KeeperException e) {
- LOG.error("Could not delete znode " + unassignedZNode, e);
- throw new IOException(e);
- } catch (InterruptedException e) {
- LOG.error("Could not delete znode " + unassignedZNode, e);
- throw new IOException(e);
- }
- }
-
- // If the UNASSIGNED ZNode does not exist, create it.
- zkWrapper.createZNodeIfNotExists(unassignedZNode);
-
- // TODO: get the outstanding changes in UNASSIGNED
-
- // Set a watch on Zookeeper's UNASSIGNED node if it exists.
- zkWrapper.registerListener(this);
- }
-
- /**
- * This is the processing loop that gets triggered from the ZooKeeperWrapper.
- * This zookeeper events process function dies the following:
- * - WATCHES the following events: NodeCreated, NodeDataChanged, NodeChildrenChanged
- * - IGNORES the following events: None, NodeDeleted
- */
- @Override
- public synchronized void process(WatchedEvent event) {
- EventType type = event.getType();
- LOG.debug("ZK-EVENT-PROCESS: Got zkEvent " + type +
- " state:" + event.getState() +
- " path:" + event.getPath());
-
- // Handle the ignored events
- if(type.equals(EventType.None) ||
- type.equals(EventType.NodeDeleted)) {
- return;
- }
-
- // check if the path is for the UNASSIGNED directory we care about
- if(event.getPath() == null ||
- !event.getPath().startsWith(zkWrapper.getZNodePathForHBase(
- zkWrapper.getRegionInTransitionZNode()))) {
- return;
- }
-
- try
- {
- /*
- * If a node is created in the UNASSIGNED directory in zookeeper, then:
- * 1. watch its updates (this is an unassigned region).
- * 2. read to see what its state is and handle as needed (state may have
- * changed before we started watching it)
- */
- if(type.equals(EventType.NodeCreated)) {
- zkWrapper.watchZNode(event.getPath());
- handleRegionStateInZK(event.getPath());
- }
- /*
- * Data on some node has changed. Read to see what the state is and handle
- * as needed.
- */
- else if(type.equals(EventType.NodeDataChanged)) {
- handleRegionStateInZK(event.getPath());
- }
- /*
- * If there were some nodes created then watch those nodes
- */
- else if(type.equals(EventType.NodeChildrenChanged)) {
- List<ZNodePathAndData> newZNodes =
- zkWrapper.watchAndGetNewChildren(event.getPath());
- for(ZNodePathAndData zNodePathAndData : newZNodes) {
- LOG.debug("Handling updates for znode: " + zNodePathAndData.getzNodePath());
- handleRegionStateInZK(zNodePathAndData.getzNodePath(),
- zNodePathAndData.getData());
- }
- }
- }
- catch (IOException e)
- {
- LOG.error("Could not process event from ZooKeeper", e);
- }
- }
-
- /**
- * Read the state of a node in ZK, and do the needful. We want to do the
- * following:
- * 1. If region's state is updated as CLOSED, invoke the ClosedRegionHandler.
- * 2. If region's state is updated as OPENED, invoke the OpenRegionHandler.
- * @param zNodePath
- * @throws IOException
- */
- private void handleRegionStateInZK(String zNodePath) throws IOException {
- byte[] data = zkWrapper.readZNode(zNodePath, null);
- handleRegionStateInZK(zNodePath, data);
- }
-
- private void handleRegionStateInZK(String zNodePath, byte[] data) {
- // a null value is set when a node is created, we don't need to handle this
- if(data == null) {
- return;
- }
- String rgnInTransitNode = zkWrapper.getRegionInTransitionZNode();
- String region = zNodePath.substring(
- zNodePath.indexOf(rgnInTransitNode) + rgnInTransitNode.length() + 1);
- HBaseEventType rsEvent = HBaseEventType.fromByte(data[0]);
-
- // if the node was CLOSED then handle it
- if(rsEvent == HBaseEventType.RS2ZK_REGION_CLOSED) {
- new MasterCloseRegionHandler(rsEvent, serverManager, serverName, region, data).submit();
- }
- // if the region was OPENED then handle that
- else if(rsEvent == HBaseEventType.RS2ZK_REGION_OPENED ||
- rsEvent == HBaseEventType.RS2ZK_REGION_OPENING) {
- new MasterOpenRegionHandler(rsEvent, serverManager, serverName, region, data).submit();
- }
- }
-}
-
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java Tue Jul 27 21:57:00 2010
@@ -23,9 +23,8 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.util.Writables;
@@ -38,16 +37,16 @@ import org.apache.hadoop.hbase.util.Writ
public class MasterCloseRegionHandler extends HBaseEventHandler
{
private static final Log LOG = LogFactory.getLog(MasterCloseRegionHandler.class);
-
+
private String regionName;
protected byte[] serializedData;
- RegionTransitionEventData hbEventData;
+ RegionTransitionData hbEventData;
ServerManager serverManager;
-
- public MasterCloseRegionHandler(HBaseEventType eventType,
- ServerManager serverManager,
- String serverName,
- String regionName,
+
+ public MasterCloseRegionHandler(HBaseEventType eventType,
+ ServerManager serverManager,
+ String serverName,
+ String regionName,
byte[] serializedData) {
super(false, serverName, eventType);
this.regionName = regionName;
@@ -56,13 +55,13 @@ public class MasterCloseRegionHandler ex
}
/**
- * Handle the various events relating to closing regions. We can get the
+ * Handle the various events relating to closing regions. We can get the
* following events here:
* - RS_REGION_CLOSING : No-op
- * - RS_REGION_CLOSED : The region is closed. If we are not in a shutdown
- * state, find the RS to open this region. This could
- * be a part of a region move, or just that the RS has
- * died. Should result in a M_REQUEST_OPENREGION event
+ * - RS_REGION_CLOSED : The region is closed. If we are not in a shutdown
+ * state, find the RS to open this region. This could
+ * be a part of a region move, or just that the RS has
+ * died. Should result in a M_REQUEST_OPENREGION event
* getting created.
*/
@Override
@@ -72,22 +71,22 @@ public class MasterCloseRegionHandler ex
// handle RS_REGION_CLOSED events
handleRegionClosedEvent();
}
-
+
private void handleRegionClosedEvent() {
try {
if(hbEventData == null) {
- hbEventData = new RegionTransitionEventData();
+ hbEventData = new RegionTransitionData();
Writables.getWritable(serializedData, hbEventData);
}
} catch (IOException e) {
LOG.error("Could not deserialize additional args for Close region", e);
}
- // process the region close - this will cause the reopening of the
+ // process the region close - this will cause the reopening of the
// region as a part of the heartbeat of some RS
serverManager.processRegionClose(hbEventData.getHmsg().getRegionInfo());
LOG.info("Processed close of region " + hbEventData.getHmsg().getRegionInfo().getRegionNameAsString());
}
-
+
public String getRegionName() {
return regionName;
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java Tue Jul 27 21:57:00 2010
@@ -26,9 +26,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.util.Writables;
@@ -46,13 +45,13 @@ public class MasterOpenRegionHandler ext
// other args passed in a byte array form
protected byte[] serializedData;
private String regionName;
- private RegionTransitionEventData hbEventData;
+ private RegionTransitionData hbEventData;
ServerManager serverManager;
- public MasterOpenRegionHandler(HBaseEventType eventType,
- ServerManager serverManager,
- String serverName,
- String regionName,
+ public MasterOpenRegionHandler(HBaseEventType eventType,
+ ServerManager serverManager,
+ String serverName,
+ String regionName,
byte[] serData) {
super(false, serverName, eventType);
this.regionName = regionName;
@@ -61,14 +60,14 @@ public class MasterOpenRegionHandler ext
}
/**
- * Handle the various events relating to opening regions. We can get the
+ * Handle the various events relating to opening regions. We can get the
* following events here:
- * - RS_REGION_OPENING : Keep track to see how long the region open takes.
- * If the RS is taking too long, then revert the
- * region back to closed state so that it can be
+ * - RS_REGION_OPENING : Keep track to see how long the region open takes.
+ * If the RS is taking too long, then revert the
+ * region back to closed state so that it can be
* re-assigned.
- * - RS_REGION_OPENED : The region is opened. Add an entry into META for
- * the RS having opened this region. Then delete this
+ * - RS_REGION_OPENED : The region is opened. Add an entry into META for
+ * the RS having opened this region. Then delete this
* entry in ZK.
*/
@Override
@@ -82,30 +81,30 @@ public class MasterOpenRegionHandler ext
handleRegionOpenedEvent();
}
}
-
+
private void handleRegionOpeningEvent() {
- // TODO: not implemented.
+ // TODO: not implemented.
LOG.debug("NO-OP call to handling region opening event");
- // Keep track to see how long the region open takes. If the RS is taking too
- // long, then revert the region back to closed state so that it can be
+ // Keep track to see how long the region open takes. If the RS is taking too
+ // long, then revert the region back to closed state so that it can be
// re-assigned.
}
private void handleRegionOpenedEvent() {
try {
if(hbEventData == null) {
- hbEventData = new RegionTransitionEventData();
+ hbEventData = new RegionTransitionData();
Writables.getWritable(serializedData, hbEventData);
}
} catch (IOException e) {
LOG.error("Could not deserialize additional args for Open region", e);
}
- LOG.debug("RS " + hbEventData.getRsName() + " has opened region " + regionName);
- HServerInfo serverInfo = serverManager.getServerInfo(hbEventData.getRsName());
+ LOG.debug("RS " + hbEventData.getServerName() + " has opened region " + regionName);
+ HServerInfo serverInfo = serverManager.getServerInfo(hbEventData.getServerName());
ArrayList<HMsg> returnMsgs = new ArrayList<HMsg>();
serverManager.processRegionOpen(serverInfo, hbEventData.getHmsg().getRegionInfo(), returnMsgs);
if(returnMsgs.size() > 0) {
- LOG.error("Open region tried to send message: " + returnMsgs.get(0).getType() +
+ LOG.error("Open region tried to send message: " + returnMsgs.get(0).getType() +
" about " + returnMsgs.get(0).getRegionInfo().getRegionNameAsString());
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=979884&r1=979883&r2=979884&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Jul 27 21:57:00 2010
@@ -2329,8 +2329,8 @@ public class HRegionServer implements HR
// ServerStatus
@Override
- public void abortServer() {
- this.abort("Received abortServer call");
+ public void abort() {
+ this.abort("Received abort call");
}
@Override