You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2013/10/02 00:02:02 UTC
svn commit: r1528227 [2/2] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/
hbase-it/src/test/java/org/apache/hadoop/hbase/
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/
hbase-it/src/test/java/org/apache/hadoop/hba...
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java Tue Oct 1 22:02:01 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -29,6 +30,7 @@ import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.Server;
@@ -78,14 +80,50 @@ public class RegionStates {
*/
private final TreeMap<HRegionInfo, ServerName> regionAssignments;
+ /**
+ * Encoded region name to server assignment map for re-assignment
+ * purpose. Contains the server a given region is last known assigned
+ * to, which has not completed log splitting, so not assignable.
+ * If a region is currently assigned, this server info in this
+ * map should be the same as that in regionAssignments.
+ * However the info in regionAssignments is cleared when the region
+ * is offline while the info in lastAssignments is cleared when
+ * the region is closed or the server is dead and processed.
+ */
+ private final HashMap<String, ServerName> lastAssignments;
+
+ /**
+ * Map a host port pair string to the latest start code
+ * of a region server which is known to be dead. It is dead
+ * to us, but server manager may not know it yet.
+ */
+ private final HashMap<String, Long> deadServers;
+
+ /**
+ * Map a dead servers to the time when log split is done.
+ * Since log splitting is not ordered, we have to remember
+ * all processed instances. The map is cleaned up based
+ * on a configured time. By default, we assume a dead
+ * server should be done with log splitting in two hours.
+ */
+ private final HashMap<ServerName, Long> processedServers;
+ private long lastProcessedServerCleanTime;
+
private final ServerManager serverManager;
private final Server server;
+ // The maximum time to keep a log split info in region states map
+ static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime";
+ static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours
+
RegionStates(final Server master, final ServerManager serverManager) {
regionStates = new HashMap<String, RegionState>();
regionsInTransition = new HashMap<String, RegionState>();
serverHoldings = new HashMap<ServerName, Set<HRegionInfo>>();
regionAssignments = new TreeMap<HRegionInfo, ServerName>();
+ lastAssignments = new HashMap<String, ServerName>();
+ processedServers = new HashMap<ServerName, Long>();
+ deadServers = new HashMap<String, Long>();
this.serverManager = serverManager;
this.server = master;
}
@@ -132,27 +170,32 @@ public class RegionStates {
}
/**
- * @return True if specified region assigned.
+ * @return True if specified region assigned, and not in transition.
*/
- public synchronized boolean isRegionAssigned(final HRegionInfo hri) {
- return regionAssignments.containsKey(hri);
+ public synchronized boolean isRegionOnline(final HRegionInfo hri) {
+ return !isRegionInTransition(hri) && regionAssignments.containsKey(hri);
}
/**
- * @return True if specified region offline.
+ * @return True if specified region offline/closed, but not in transition.
+ * If the region is not in the map, it is offline to us too.
*/
public synchronized boolean isRegionOffline(final HRegionInfo hri) {
- return !isRegionInTransition(hri) && isRegionInState(hri, State.OFFLINE);
+ return getRegionState(hri) == null || (!isRegionInTransition(hri)
+ && isRegionInState(hri, State.OFFLINE, State.CLOSED));
}
/**
- * @return True if specified region is in specified state
+ * @return True if specified region is in one of the specified states.
*/
public synchronized boolean isRegionInState(
- final HRegionInfo hri, final State state) {
+ final HRegionInfo hri, final State... states) {
RegionState regionState = getRegionState(hri);
State s = regionState != null ? regionState.getState() : null;
- return s == state;
+ for (State state: states) {
+ if (s == state) return true;
+ }
+ return false;
}
/**
@@ -252,8 +295,8 @@ public class RegionStates {
ServerName newServerName = serverName;
if (serverName != null &&
(state == State.CLOSED || state == State.OFFLINE)) {
- LOG.warn("Closed " + hri.getShortNameToLog() + " still on "
- + serverName + "? Ignored, reset it to null");
+ LOG.info(hri.getShortNameToLog() + " is " + state
+ + ", reset server info from " + serverName + " to null");
newServerName = null;
}
@@ -274,6 +317,20 @@ public class RegionStates {
regionsInTransition.put(regionName, regionState);
}
+ // For these states, region should be properly closed.
+ // There should be no log splitting issue.
+ if ((state == State.CLOSED || state == State.MERGED
+ || state == State.SPLIT) && lastAssignments.containsKey(regionName)) {
+ ServerName oldServerName = oldState == null ? null : oldState.getServerName();
+ ServerName last = lastAssignments.get(regionName);
+ if (last.equals(oldServerName)) {
+ lastAssignments.remove(regionName);
+ } else {
+ LOG.warn(regionName + " moved to " + state + " on "
+ + oldServerName + ", expected " + last);
+ }
+ }
+
// notify the change
this.notifyAll();
return regionState;
@@ -286,6 +343,16 @@ public class RegionStates {
*/
public synchronized void regionOnline(
final HRegionInfo hri, final ServerName serverName) {
+ if (!serverManager.isServerOnline(serverName)) {
+ // This is possible if the region server dies before master gets a
+ // chance to handle ZK event in time. At this time, if the dead server
+ // is already processed by SSH, we should ignore this event.
+ // If not processed yet, ignore and let SSH deal with it.
+ LOG.warn("Ignored, " + hri.getEncodedName()
+ + " was opened on a dead server: " + serverName);
+ return;
+ }
+
String regionName = hri.getEncodedName();
RegionState oldState = regionStates.get(regionName);
if (oldState == null) {
@@ -301,6 +368,7 @@ public class RegionStates {
updateRegionState(hri, State.OPEN, serverName);
regionsInTransition.remove(regionName);
+ lastAssignments.put(regionName, serverName);
ServerName oldServerName = regionAssignments.put(hri, serverName);
if (!serverName.equals(oldServerName)) {
LOG.info("Onlined " + hri.getShortNameToLog() + " on " + serverName);
@@ -312,12 +380,53 @@ public class RegionStates {
regions.add(hri);
if (oldServerName != null) {
LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
- serverHoldings.get(oldServerName).remove(hri);
+ Set<HRegionInfo> oldRegions = serverHoldings.get(oldServerName);
+ oldRegions.remove(hri);
+ if (oldRegions.isEmpty()) {
+ serverHoldings.remove(oldServerName);
+ }
}
}
}
/**
+ * A dead server's hlogs have been split so that all the regions
+ * used to be open on it can be safely assigned now. Mark them assignable.
+ */
+ public synchronized void logSplit(final ServerName serverName) {
+ for (Iterator<Map.Entry<String, ServerName>> it
+ = lastAssignments.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<String, ServerName> e = it.next();
+ if (e.getValue().equals(serverName)) {
+ it.remove();
+ }
+ }
+ long now = System.currentTimeMillis();
+ processedServers.put(serverName, Long.valueOf(now));
+ Configuration conf = server.getConfiguration();
+ long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME);
+ // Doesn't have to be very accurate about the clean up time
+ if (now > lastProcessedServerCleanTime + obsoleteTime) {
+ lastProcessedServerCleanTime = now;
+ long cutoff = now - obsoleteTime;
+ for (Iterator<Map.Entry<ServerName, Long>> it
+ = processedServers.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<ServerName, Long> e = it.next();
+ if (e.getValue().longValue() < cutoff) {
+ it.remove();
+ }
+ }
+ }
+ }
+
+ /**
+ * Log split is done for a given region, so it is assignable now.
+ */
+ public synchronized void logSplit(final HRegionInfo region) {
+ lastAssignments.remove(region.getEncodedName());
+ }
+
+ /**
* A region is offline, won't be in transition any more.
*/
public void regionOffline(final HRegionInfo hri) {
@@ -361,7 +470,11 @@ public class RegionStates {
ServerName oldServerName = regionAssignments.remove(hri);
if (oldServerName != null) {
LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
- serverHoldings.get(oldServerName).remove(hri);
+ Set<HRegionInfo> oldRegions = serverHoldings.get(oldServerName);
+ oldRegions.remove(hri);
+ if (oldRegions.isEmpty()) {
+ serverHoldings.remove(oldServerName);
+ }
}
}
@@ -370,19 +483,38 @@ public class RegionStates {
*/
public synchronized List<HRegionInfo> serverOffline(
final ZooKeeperWatcher watcher, final ServerName sn) {
- // Clean up this server from map of servers to regions, and remove all regions
- // of this server from online map of regions.
+ // Offline all regions on this server not already in transition.
List<HRegionInfo> rits = new ArrayList<HRegionInfo>();
- Set<HRegionInfo> assignedRegions = serverHoldings.remove(sn);
+ Set<HRegionInfo> assignedRegions = serverHoldings.get(sn);
if (assignedRegions == null) {
assignedRegions = new HashSet<HRegionInfo>();
}
+ // Offline regions outside the loop to avoid ConcurrentModificationException
+ Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
for (HRegionInfo region : assignedRegions) {
- regionAssignments.remove(region);
+ // Offline open regions, no need to offline if SPLIT/MERGED/OFFLINE
+ if (isRegionOnline(region)) {
+ regionsToOffline.add(region);
+ } else {
+ RegionState state = getRegionState(region);
+ if (state.isSplitting() || state.isMerging()) {
+ LOG.debug("Offline splitting/merging region " + state);
+ try {
+ // Delete the ZNode if exists
+ ZKAssign.deleteNodeFailSilent(watcher, region);
+ regionsToOffline.add(region);
+ } catch (KeeperException ke) {
+ server.abort("Unexpected ZK exception deleting node " + region, ke);
+ }
+ }
+ }
+ }
+
+ for (HRegionInfo hri : regionsToOffline) {
+ regionOffline(hri);
}
- Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
for (RegionState state : regionsInTransition.values()) {
HRegionInfo hri = state.getRegion();
if (assignedRegions.contains(hri)) {
@@ -390,36 +522,22 @@ public class RegionStates {
// This region must be moving away from this server, or splitting/merging.
// SSH will handle it, either skip assigning, or re-assign.
LOG.info("Transitioning " + state + " will be handled by SSH for " + sn);
- if (state.isSplitting() || state.isMerging()) {
- LOG.info("Offline splitting/merging region " + state);
- try {
- // Delete the ZNode if exists
- ZKAssign.deleteNodeFailSilent(watcher, hri);
- // Offline regions outside the loop to avoid ConcurrentModificationException
- regionsToOffline.add(hri);
- } catch (KeeperException ke) {
- server.abort("Unexpected ZK exception deleting node " + hri, ke);
- }
- }
} else if (sn.equals(state.getServerName())) {
// Region is in transition on this region server, and this
// region is not open on this server. So the region must be
// moving to this server from another one (i.e. opening or
- // pending open on this server, was open on another one
- if (state.isPendingOpen() || state.isOpening()) {
+ // pending open on this server, was open on another one.
+ // It could be in failed_close state too if tried several times
+ // to open it while the server is not reachable.
+ if (state.isPendingOpenOrOpening() || state.isFailedClose()) {
LOG.info("Found opening region " + state + " to be reassigned by SSH for " + sn);
rits.add(hri);
} else {
- LOG.warn("THIS SHOULD NOT HAPPEN: unexpected state "
- + state + " of region in transition on server " + sn);
+ LOG.warn("THIS SHOULD NOT HAPPEN: unexpected " + state);
}
}
}
- for (HRegionInfo hri : regionsToOffline) {
- regionOffline(hri);
- }
- assignedRegions.clear();
this.notifyAll();
return rits;
}
@@ -470,24 +588,58 @@ public class RegionStates {
}
/**
- * Waits until the specified region has completed assignment.
- * <p>
- * If the region is already assigned, returns immediately. Otherwise, method
- * blocks until the region is assigned.
- */
- public synchronized void waitForAssignment(
- final HRegionInfo hri) throws InterruptedException {
- if (!isRegionAssigned(hri)) return;
-
- while(!server.isStopped() && !isRegionAssigned(hri)) {
- RegionState rs = getRegionState(hri);
- LOG.info("Waiting on " + rs + " to be assigned");
- waitForUpdate(100);
- }
-
- if (server.isStopped()) {
- LOG.info("Giving up wait on region " +
- "assignment because stoppable.isStopped is set");
+ * Checking if a region was assigned to a server which is not online now.
+ * If so, we should hold re-assign this region till SSH has split its hlogs.
+ * Once logs are split, the last assignment of this region will be reset,
+ * which means a null last assignment server is ok for re-assigning.
+ *
+ * A region server could be dead but we don't know it yet. We may
+ * think it's online falsely. Therefore if a server is online, we still
+ * need to confirm it reachable and having the expected start code.
+ */
+ synchronized boolean wasRegionOnDeadServer(final String regionName) {
+ ServerName server = lastAssignments.get(regionName);
+ return isServerDeadAndNotProcessed(server);
+ }
+
+ synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
+ if (server == null) return false;
+ if (serverManager.isServerOnline(server)) {
+ String hostAndPort = server.getHostAndPort();
+ long startCode = server.getStartcode();
+ Long deadCode = deadServers.get(hostAndPort);
+ if (deadCode == null || startCode > deadCode.longValue()) {
+ if (serverManager.isServerReachable(server)) {
+ return false;
+ }
+ // The size of deadServers won't grow unbounded.
+ deadServers.put(hostAndPort, Long.valueOf(startCode));
+ }
+ // Watch out! If the server is not dead, the region could
+ // remain unassigned. That's why ServerManager#isServerReachable
+ // should use some retry.
+ //
+ // We cache this info since it is very unlikely for that
+ // instance to come back up later on. We don't want to expire
+ // the server since we prefer to let it die naturally.
+ LOG.warn("Couldn't reach online server " + server);
+ }
+ // Now, we know it's dead. Check if it's processed
+ return !processedServers.containsKey(server);
+ }
+
+ /**
+ * Get the last region server a region was on for purpose of re-assignment,
+ * i.e. should the re-assignment be held back till log split is done?
+ */
+ synchronized ServerName getLastRegionServerOfRegion(final String regionName) {
+ return lastAssignments.get(regionName);
+ }
+
+ synchronized void setLastRegionServerOfRegions(
+ final ServerName serverName, final List<HRegionInfo> regionInfos) {
+ for (HRegionInfo hri: regionInfos) {
+ lastAssignments.put(hri.getEncodedName(), serverName);
}
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Oct 1 22:02:01 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Triple;
@@ -180,6 +181,7 @@ public class ServerManager {
this(master, services, true);
}
+ @SuppressWarnings("deprecation")
ServerManager(final Server master, final MasterServices services,
final boolean connect) throws IOException {
this.master = master;
@@ -722,6 +724,29 @@ public class ServerManager {
ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
}
+ /**
+ * Check if a region server is reachable and has the expected start code
+ */
+ public boolean isServerReachable(ServerName server) {
+ if (server == null) throw new NullPointerException("Passed server is null");
+ int maximumAttempts = Math.max(1, master.getConfiguration().getInt(
+ "hbase.master.maximum.ping.server.attempts", 10));
+ for (int i = 0; i < maximumAttempts; i++) {
+ try {
+ AdminService.BlockingInterface admin = getRsAdmin(server);
+ if (admin != null) {
+ ServerInfo info = ProtobufUtil.getServerInfo(admin);
+ return info != null && info.hasServerName()
+ && server.getStartcode() == info.getServerName().getStartCode();
+ }
+ } catch (IOException ioe) {
+ LOG.debug("Couldn't reach " + server + ", try=" + i
+ + " of " + maximumAttempts, ioe);
+ }
+ }
+ return false;
+ }
+
/**
* @param sn
* @return Admin interface for the remote regionserver named <code>sn</code>
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java Tue Oct 1 22:02:01 2013
@@ -98,7 +98,7 @@ public class ClosedRegionHandler extends
}
// ZK Node is in CLOSED state, assign it.
assignmentManager.getRegionStates().updateRegionState(
- regionInfo, RegionState.State.CLOSED, null);
+ regionInfo, RegionState.State.CLOSED);
// This below has to do w/ online enable/disable of a table
assignmentManager.removeClosedRegion(regionInfo);
assignmentManager.assign(regionInfo, true);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Tue Oct 1 22:02:01 2013
@@ -222,7 +222,7 @@ public class EnableTableHandler extends
for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
HRegionInfo hri = regionLocation.getFirst();
ServerName sn = regionLocation.getSecond();
- if (!regionStates.isRegionInTransition(hri) && !regionStates.isRegionAssigned(hri)) {
+ if (regionStates.isRegionOffline(hri)) {
if (this.retainAssignment && sn != null && serverManager.isServerOnline(sn)) {
this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java Tue Oct 1 22:02:01 2013
@@ -19,9 +19,7 @@
package org.apache.hadoop.hbase.master.handler;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -64,7 +62,8 @@ public class MetaServerShutdownHandler e
} else {
this.services.getMasterFileSystem().splitMetaLog(serverName);
}
- }
+ am.getRegionStates().logSplit(HRegionInfo.FIRST_META_REGIONINFO);
+ }
} catch (IOException ioe) {
this.services.getExecutorService().submit(this);
this.deadServers.add(serverName);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java Tue Oct 1 22:02:01 2013
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
+import java.util.concurrent.locks.Lock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -141,8 +142,9 @@ public class ServerShutdownHandler exten
// If AssignmentManager hasn't finished rebuilding user regions,
// we are not ready to assign dead regions either. So we re-queue up
// the dead server for further processing too.
+ AssignmentManager am = services.getAssignmentManager();
if (isCarryingMeta() // hbase:meta
- || !services.getAssignmentManager().isFailoverCleanupDone()) {
+ || !am.isFailoverCleanupDone()) {
this.services.getServerManager().processDeadServer(serverName, this.shouldSplitHlog);
return;
}
@@ -174,7 +176,7 @@ public class ServerShutdownHandler exten
throw new IOException("Interrupted", e);
} catch (IOException ioe) {
LOG.info("Received exception accessing hbase:meta during server shutdown of " +
- serverName + ", retrying hbase:meta read", ioe);
+ serverName + ", retrying hbase:meta read", ioe);
}
}
if (this.server.isStopped()) {
@@ -192,6 +194,7 @@ public class ServerShutdownHandler exten
} else {
this.services.getMasterFileSystem().splitLog(serverName);
}
+ am.getRegionStates().logSplit(serverName);
} else {
LOG.info("Skipping log splitting for " + serverName);
}
@@ -203,7 +206,6 @@ public class ServerShutdownHandler exten
// doing after log splitting. Could do some states before -- OPENING?
// OFFLINE? -- and then others after like CLOSING that depend on log
// splitting.
- AssignmentManager am = services.getAssignmentManager();
List<HRegionInfo> regionsInTransition = am.processServerShutdown(serverName);
LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) +
" region(s) that " + (serverName == null? "null": serverName) +
@@ -221,52 +223,56 @@ public class ServerShutdownHandler exten
if (regionsInTransition.contains(hri)) {
continue;
}
- RegionState rit = regionStates.getRegionTransitionState(hri);
- if (processDeadRegion(hri, e.getValue(), am, server.getCatalogTracker())) {
- ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri);
- if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
- // If this region is in transition on the dead server, it must be
- // opening or pending_open, which should have been covered by AM#processServerShutdown
- LOG.info("Skip assigning region " + hri.getRegionNameAsString()
- + " because it has been opened in " + addressFromAM.getServerName());
- continue;
- }
- if (rit != null) {
- if (!rit.isOnServer(serverName)
- || rit.isClosed() || rit.isOpened()) {
- // Skip regions that are in transition on other server,
- // or in state closed/opened
- LOG.info("Skip assigning region " + rit);
+ String encodedName = hri.getEncodedName();
+ Lock lock = am.acquireRegionLock(encodedName);
+ try {
+ RegionState rit = regionStates.getRegionTransitionState(hri);
+ if (processDeadRegion(hri, e.getValue(), am, server.getCatalogTracker())) {
+ ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri);
+ if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
+ // If this region is in transition on the dead server, it must be
+ // opening or pending_open, which should have been covered by AM#processServerShutdown
+ LOG.info("Skip assigning region " + hri.getRegionNameAsString()
+ + " because it has been opened in " + addressFromAM.getServerName());
continue;
}
- try{
- //clean zk node
- LOG.info("Reassigning region with rs = " + rit + " and deleting zk node if exists");
- ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), hri);
- } catch (KeeperException ke) {
- this.server.abort("Unexpected ZK exception deleting unassigned node " + hri, ke);
- return;
+ if (rit != null) {
+ if (rit.getServerName() != null && !rit.isOnServer(serverName)) {
+ // Skip regions that are in transition on other server
+ LOG.info("Skip assigning region in transition on other server" + rit);
+ continue;
+ }
+ try{
+ //clean zk node
+ LOG.info("Reassigning region with rs = " + rit + " and deleting zk node if exists");
+ ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), hri);
+ } catch (KeeperException ke) {
+ this.server.abort("Unexpected ZK exception deleting unassigned node " + hri, ke);
+ return;
+ }
+ }
+ toAssignRegions.add(hri);
+ } else if (rit != null) {
+ if (rit.isPendingCloseOrClosing()
+ && am.getZKTable().isDisablingOrDisabledTable(hri.getTable())) {
+ // If the table was partially disabled and the RS went down, we should clear the RIT
+ // and remove the node for the region.
+ // The rit that we use may be stale in case the table was in DISABLING state
+ // but though we did assign we will not be clearing the znode in CLOSING state.
+ // Doing this will have no harm. See HBASE-5927
+ am.deleteClosingOrClosedNode(hri);
+ am.regionOffline(hri);
+ } else {
+ LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition "
+ + rit + " not to be assigned by SSH of server " + serverName);
}
}
- toAssignRegions.add(hri);
- } else if (rit != null) {
- if ((rit.isClosing() || rit.isPendingClose())
- && am.getZKTable().isDisablingOrDisabledTable(hri.getTable())) {
- // If the table was partially disabled and the RS went down, we should clear the RIT
- // and remove the node for the region.
- // The rit that we use may be stale in case the table was in DISABLING state
- // but though we did assign we will not be clearing the znode in CLOSING state.
- // Doing this will have no harm. See HBASE-5927
- am.deleteClosingOrClosedNode(hri);
- am.regionOffline(hri);
- } else {
- LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition "
- + rit + " not to be assigned by SSH of server " + serverName);
- }
+ } finally {
+ lock.unlock();
}
}
}
-
+
try {
am.assign(toAssignRegions);
} catch (InterruptedException ie) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Tue Oct 1 22:02:01 2013
@@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.client.Ro
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -331,6 +332,18 @@ public class HBaseFsck extends Configure
}
}
+ errors.print("Average load: " + status.getAverageLoad());
+ errors.print("Number of requests: " + status.getRequestsCount());
+ errors.print("Number of regions: " + status.getRegionsCount());
+
+ Map<String, RegionState> rits = status.getRegionsInTransition();
+ errors.print("Number of regions in transition: " + rits.size());
+ if (details) {
+ for (RegionState state: rits.values()) {
+ errors.print(" " + state.toDescriptiveString());
+ }
+ }
+
// Determine what's deployed
processRegionServers(regionServers);
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java Tue Oct 1 22:02:01 2013
@@ -150,7 +150,7 @@ public class TestDrainingServer {
am.waitForAssignment(REGIONINFO);
- assertTrue(am.getRegionStates().isRegionAssigned(REGIONINFO));
+ assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java Tue Oct 1 22:02:01 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
@@ -58,6 +59,7 @@ import org.apache.hadoop.hbase.util.Envi
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -75,15 +77,15 @@ public class TestAssignmentManagerOnClus
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- // Using the mock load balancer to control region plans
+ // Using the our load balancer to control region plans
conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
- MockLoadBalancer.class, LoadBalancer.class);
+ MyLoadBalancer.class, LoadBalancer.class);
conf.setClass(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
- MockRegionObserver.class, RegionObserver.class);
+ MyRegionObserver.class, RegionObserver.class);
// Reduce the maximum attempts to speed up the test
conf.setInt("hbase.assignment.maximum.attempts", 3);
- TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniCluster(1, 4, null, MyMaster.class, null);
admin = TEST_UTIL.getHBaseAdmin();
}
@@ -306,12 +308,12 @@ public class TestAssignmentManagerOnClus
AssignmentManager am = master.getAssignmentManager();
assertTrue(am.waitForAssignment(hri));
- MockRegionObserver.preCloseEnabled.set(true);
+ MyRegionObserver.preCloseEnabled.set(true);
am.unassign(hri);
RegionState state = am.getRegionStates().getRegionState(hri);
assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
- MockRegionObserver.preCloseEnabled.set(false);
+ MyRegionObserver.preCloseEnabled.set(false);
am.unassign(hri, true);
// region is closing now, will be re-assigned automatically.
@@ -327,7 +329,7 @@ public class TestAssignmentManagerOnClus
getRegionStates().getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
} finally {
- MockRegionObserver.preCloseEnabled.set(false);
+ MyRegionObserver.preCloseEnabled.set(false);
TEST_UTIL.deleteTable(Bytes.toBytes(table));
}
}
@@ -353,12 +355,12 @@ public class TestAssignmentManagerOnClus
AssignmentManager am = master.getAssignmentManager();
assertTrue(am.waitForAssignment(hri));
- MockRegionObserver.preCloseEnabled.set(true);
+ MyRegionObserver.preCloseEnabled.set(true);
am.unassign(hri);
RegionState state = am.getRegionStates().getRegionState(hri);
assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
- MockRegionObserver.preCloseEnabled.set(false);
+ MyRegionObserver.preCloseEnabled.set(false);
am.unassign(hri, true);
// region may still be assigned now since it's closing,
@@ -371,7 +373,7 @@ public class TestAssignmentManagerOnClus
getRegionStates().getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
} finally {
- MockRegionObserver.preCloseEnabled.set(false);
+ MyRegionObserver.preCloseEnabled.set(false);
TEST_UTIL.deleteTable(Bytes.toBytes(table));
}
}
@@ -392,7 +394,7 @@ public class TestAssignmentManagerOnClus
desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
MetaEditor.addRegionToMeta(meta, hri);
- MockLoadBalancer.controledRegion = hri.getEncodedName();
+ MyLoadBalancer.controledRegion = hri.getEncodedName();
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
master.assignRegion(hri);
@@ -404,7 +406,7 @@ public class TestAssignmentManagerOnClus
// Failed to open since no plan, so it's on no server
assertNull(state.getServerName());
- MockLoadBalancer.controledRegion = null;
+ MyLoadBalancer.controledRegion = null;
master.assignRegion(hri);
assertTrue(am.waitForAssignment(hri));
@@ -412,7 +414,7 @@ public class TestAssignmentManagerOnClus
getRegionStates().getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
} finally {
- MockLoadBalancer.controledRegion = null;
+ MyLoadBalancer.controledRegion = null;
TEST_UTIL.deleteTable(Bytes.toBytes(table));
}
}
@@ -535,7 +537,7 @@ public class TestAssignmentManagerOnClus
AssignmentManager am = master.getAssignmentManager();
assertTrue(am.waitForAssignment(hri));
- MockRegionObserver.postCloseEnabled.set(true);
+ MyRegionObserver.postCloseEnabled.set(true);
am.unassign(hri);
// Now region should pending_close or closing
// Unassign it again forcefully so that we can trigger already
@@ -547,7 +549,7 @@ public class TestAssignmentManagerOnClus
// Let region closing move ahead. The region should be closed
// properly and re-assigned automatically
- MockRegionObserver.postCloseEnabled.set(false);
+ MyRegionObserver.postCloseEnabled.set(false);
// region may still be assigned now since it's closing,
// let's check if it's assigned after it's out of transition
@@ -559,7 +561,7 @@ public class TestAssignmentManagerOnClus
getRegionStates().getRegionServerOfRegion(hri);
TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
} finally {
- MockRegionObserver.postCloseEnabled.set(false);
+ MyRegionObserver.postCloseEnabled.set(false);
TEST_UTIL.deleteTable(Bytes.toBytes(table));
}
}
@@ -580,14 +582,14 @@ public class TestAssignmentManagerOnClus
desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
MetaEditor.addRegionToMeta(meta, hri);
- MockRegionObserver.postOpenEnabled.set(true);
- MockRegionObserver.postOpenCalled = false;
+ MyRegionObserver.postOpenEnabled.set(true);
+ MyRegionObserver.postOpenCalled = false;
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
// Region will be opened, but it won't complete
master.assignRegion(hri);
long end = EnvironmentEdgeManager.currentTimeMillis() + 20000;
// Wait till postOpen is called
- while (!MockRegionObserver.postOpenCalled ) {
+ while (!MyRegionObserver.postOpenCalled ) {
assertFalse("Timed out waiting for postOpen to be called",
EnvironmentEdgeManager.currentTimeMillis() > end);
Thread.sleep(300);
@@ -604,7 +606,7 @@ public class TestAssignmentManagerOnClus
// Let's forcefully re-assign it to trigger closing/opening
// racing. This test is to make sure this scenario
// is handled properly.
- MockRegionObserver.postOpenEnabled.set(false);
+ MyRegionObserver.postOpenEnabled.set(false);
am.assign(hri, true, true);
// let's check if it's assigned after it's out of transition
@@ -617,12 +619,124 @@ public class TestAssignmentManagerOnClus
assertFalse("Region should assigned on a new region server",
oldServerName.equals(serverName));
} finally {
- MockRegionObserver.postOpenEnabled.set(false);
+ MyRegionObserver.postOpenEnabled.set(false);
TEST_UTIL.deleteTable(Bytes.toBytes(table));
}
}
- static class MockLoadBalancer extends StochasticLoadBalancer {
+ /**
+ * Test force unassign/assign a region hosted on a dead server
+ */
+ @Test (timeout=60000)
+ public void testAssignRacingWithSSH() throws Exception {
+ String table = "testAssignRacingWithSSH";
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ MyMaster master = null;
+ try {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
+ desc.addFamily(new HColumnDescriptor(FAMILY));
+ admin.createTable(desc);
+
+ HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
+ HRegionInfo hri = new HRegionInfo(
+ desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
+ MetaEditor.addRegionToMeta(meta, hri);
+
+ // Assign the region
+ master = (MyMaster)cluster.getMaster();
+ master.assignRegion(hri);
+
+ // Hold SSH before killing the hosting server
+ master.enableSSH(false);
+
+ // Kill the hosting server
+ AssignmentManager am = master.getAssignmentManager();
+ RegionStates regionStates = am.getRegionStates();
+ assertTrue(am.waitForAssignment(hri));
+ RegionState state = regionStates.getRegionState(hri);
+ ServerName oldServerName = state.getServerName();
+ cluster.killRegionServer(oldServerName);
+ cluster.waitForRegionServerToStop(oldServerName, -1);
+
+ // You can't assign a dead region before SSH
+ am.assign(hri, true, true);
+ state = regionStates.getRegionState(hri);
+ assertTrue(state.isFailedClose());
+
+ // You can't unassign a dead region before SSH either
+ am.unassign(hri, true);
+ state = regionStates.getRegionState(hri);
+ assertTrue(state.isFailedClose());
+
+ synchronized (regionStates) {
+ // Enable SSH so that log can be split
+ master.enableSSH(true);
+
+ // We hold regionStates now, so logSplit
+ // won't be known to AM yet.
+ am.unassign(hri, true);
+ state = regionStates.getRegionState(hri);
+ assertTrue(state.isOffline());
+ }
+
+ // let's check if it's assigned after it's out of transition.
+ // no need to assign it manually, SSH should do it
+ am.waitOnRegionToClearRegionsInTransition(hri);
+ assertTrue(am.waitForAssignment(hri));
+
+ ServerName serverName = master.getAssignmentManager().
+ getRegionStates().getRegionServerOfRegion(hri);
+ TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
+ } finally {
+ if (master != null) {
+ master.enableSSH(true);
+ }
+ TEST_UTIL.deleteTable(Bytes.toBytes(table));
+ }
+ }
+
+ /**
+ * Test force unassign/assign a region of a disabled table
+ */
+ @Test (timeout=60000)
+ public void testAssignDisabledRegion() throws Exception {
+ String table = "testAssignDisabledRegion";
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ MyMaster master = null;
+ try {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
+ desc.addFamily(new HColumnDescriptor(FAMILY));
+ admin.createTable(desc);
+
+ HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
+ HRegionInfo hri = new HRegionInfo(
+ desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
+ MetaEditor.addRegionToMeta(meta, hri);
+
+ // Assign the region
+ master = (MyMaster)cluster.getMaster();
+ master.assignRegion(hri);
+ AssignmentManager am = master.getAssignmentManager();
+ RegionStates regionStates = am.getRegionStates();
+ assertTrue(am.waitForAssignment(hri));
+
+ // Disable the table
+ admin.disableTable(table);
+ assertTrue(regionStates.isRegionOffline(hri));
+
+ // You can't assign a disabled region
+ am.assign(hri, true, true);
+ assertTrue(regionStates.isRegionOffline(hri));
+
+ // You can't unassign a disabled region either
+ am.unassign(hri, true);
+ assertTrue(regionStates.isRegionOffline(hri));
+ } finally {
+ TEST_UTIL.deleteTable(Bytes.toBytes(table));
+ }
+ }
+
+ static class MyLoadBalancer extends StochasticLoadBalancer {
// For this region, if specified, always assign to nowhere
static volatile String controledRegion = null;
@@ -636,7 +750,28 @@ public class TestAssignmentManagerOnClus
}
}
- public static class MockRegionObserver extends BaseRegionObserver {
+ public static class MyMaster extends HMaster {
+ AtomicBoolean enabled = new AtomicBoolean(true);
+
+ public MyMaster(Configuration conf) throws IOException, KeeperException,
+ InterruptedException {
+ super(conf);
+ }
+
+ @Override
+ public boolean isServerShutdownHandlerEnabled() {
+ return enabled.get() && super.isServerShutdownHandlerEnabled();
+ }
+
+ public void enableSSH(boolean enabled) {
+ this.enabled.set(enabled);
+ if (enabled) {
+ serverManager.processQueuedDeadServers();
+ }
+ }
+ }
+
+ public static class MyRegionObserver extends BaseRegionObserver {
// If enabled, fail all preClose calls
static AtomicBoolean preCloseEnabled = new AtomicBoolean(false);
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java Tue Oct 1 22:02:01 2013
@@ -167,6 +167,7 @@ public class TestSplitTransactionOnClust
return hri;
}
+ @SuppressWarnings("deprecation")
@Test(timeout = 60000)
public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
final TableName tableName =
@@ -209,6 +210,8 @@ public class TestSplitTransactionOnClust
SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
try {
secondSplit = true;
+ // make region splittable
+ region.initialize();
st.prepare();
st.execute(regionServer, regionServer);
} catch (IOException e) {
@@ -933,8 +936,8 @@ public class TestSplitTransactionOnClust
assertTrue("not able to find a splittable region", region != null);
SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
@Override
- int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
- throws KeeperException, IOException {
+ int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region,
+ ServerName serverName) throws KeeperException, IOException {
throw new SplittingNodeCreationFailedException ();
}
};