You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/11/12 00:36:48 UTC
svn commit: r1201104 - in /hbase/trunk: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Author: stack
Date: Fri Nov 11 23:36:47 2011
New Revision: 1201104
URL: http://svn.apache.org/viewvc?rev=1201104&view=rev
Log:
HBASE-4511 There is data loss when master failovers
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1201104&r1=1201103&r2=1201104&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Nov 11 23:36:47 2011
@@ -469,6 +469,7 @@ Release 0.92.0 - Unreleased
HBASE-4734 [bulk load] Warn if bulk load directory contained no files
HBASE-4723 Loads of NotAllMetaRegionsOnlineException traces when starting
the master
+ HBASE-4511 There is data loss when master failovers
TESTS
HBASE-4450 test for number of blocks read: to serve as baseline for expected
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java?rev=1201104&r1=1201103&r2=1201104&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java Fri Nov 11 23:36:47 2011
@@ -268,6 +268,21 @@ public class CatalogTracker {
}
/**
+ * Method used by master on startup trying to figure state of cluster.
+ * Returns the current meta location unless its null. In this latter case,
+ * it has not yet been set so go check whats up in <code>-ROOT-</code> and
+ * return that.
+ * @return{@link ServerName} for server hosting <code>.META.</code> or if null,
+ * we'll read the location that is up in <code>-ROOT-</code> table (which
+ * could be null or just plain stale).
+ * @throws IOException
+ */
+ public ServerName getMetaLocationOrReadLocationFromRoot() throws IOException {
+ ServerName sn = getMetaLocation();
+ return sn != null? sn: MetaReader.getMetaRegionLocation(this);
+ }
+
+ /**
* Waits indefinitely for availability of <code>-ROOT-</code>. Used during
* cluster startup.
* @throws InterruptedException if interrupted while waiting
@@ -336,7 +351,7 @@ public class CatalogTracker {
* @return connection to server hosting root
* @throws NotAllMetaRegionsOnlineException if timed out waiting
* @throws IOException
- * @deprecated Use {@link #getRootServerConnection(long)}
+ * @deprecated Use #getRootServerConnection(long)
*/
public HRegionInterface waitForRootServerConnectionDefault()
throws NotAllMetaRegionsOnlineException, IOException {
@@ -381,8 +396,7 @@ public class CatalogTracker {
// Now read the current .META. content from -ROOT-. Note: This goes via
// an HConnection. It has its own way of figuring root and meta locations
// which we have to wait on.
- ServerName newLocation =
- MetaReader.readRegionLocation(this, META_REGION_NAME);
+ ServerName newLocation = MetaReader.getMetaRegionLocation(this);
if (newLocation == null) return null;
HRegionInterface newConnection = getCachedConnection(newLocation);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1201104&r1=1201103&r2=1201104&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Fri Nov 11 23:36:47 2011
@@ -193,6 +193,8 @@ public class MetaReader {
throws IOException {
// Passing the CatalogTracker's connection configuration ensures this
// HTable instance uses the CatalogTracker's connection.
+ org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
+ if (c == null) throw new NullPointerException("No connection");
return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
}
@@ -251,7 +253,7 @@ public class MetaReader {
* @param metaServer connection to server hosting ROOT
* @return location of META in ROOT where location, or null if not available
* @throws IOException
- * @deprecated Does not retry; use {@link #readRegionLocation(CatalogTracker, byte[])}
+ * @deprecated Does not retry; use #getMetaRegionLocation(CatalogTracker)
*/
public static ServerName readMetaLocation(HRegionInterface metaServer)
throws IOException {
@@ -260,13 +262,26 @@ public class MetaReader {
}
/**
+ * Gets the location of <code>.META.</code> region by reading content of
+ * <code>-ROOT-</code>.
+ * @param ct
+ * @return location of <code>.META.</code> region as a {@link ServerName} or
+ * null if not found
+ * @throws IOException
+ */
+ static ServerName getMetaRegionLocation(final CatalogTracker ct)
+ throws IOException {
+ return MetaReader.readRegionLocation(ct, CatalogTracker.META_REGION_NAME);
+ }
+
+ /**
* Reads the location of the specified region
* @param catalogTracker
* @param regionName region whose location we are after
* @return location of region as a {@link ServerName} or null if not found
* @throws IOException
*/
- public static ServerName readRegionLocation(CatalogTracker catalogTracker,
+ static ServerName readRegionLocation(CatalogTracker catalogTracker,
byte [] regionName)
throws IOException {
Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1201104&r1=1201103&r2=1201104&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri Nov 11 23:36:47 2011
@@ -462,7 +462,8 @@ implements HMasterInterface, HMasterRegi
for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
if (!this.serverManager.isServerOnline(sn)) {
// Not registered; add it.
- LOG.info("Registering server found up in zk: " + sn);
+ LOG.info("Registering server found up in zk but who has not yet " +
+ "reported in: " + sn);
this.serverManager.recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
}
}
@@ -526,14 +527,23 @@ implements HMasterInterface, HMasterRegi
status.setStatus("Assigning ROOT region");
boolean rit = this.assignmentManager.
processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO);
+ ServerName expiredServer = null;
if (!catalogTracker.verifyRootRegionLocation(timeout)) {
- this.assignmentManager.assignRoot();
+ ServerName currentRootServer = this.catalogTracker.getRootLocation();
+ if (expireIfOnline(currentRootServer)) {
+ // We are expiring this server. The processing of expiration will assign
+ // root so don't do it here.
+ expiredServer = currentRootServer;
+ } else {
+ // Root was not on an online server when we failed verification
+ this.assignmentManager.assignRoot();
+ }
this.catalogTracker.waitForRoot();
//This guarantees that the transition has completed
this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
assigned++;
} else {
- // Region already assigned. We didnt' assign it. Add to in-memory state.
+ // Region already assigned. We didn't assign it. Add to in-memory state.
this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO,
this.catalogTracker.getRootLocation());
}
@@ -545,7 +555,15 @@ implements HMasterInterface, HMasterRegi
rit = this.assignmentManager.
processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
if (!this.catalogTracker.verifyMetaRegionLocation(timeout)) {
- this.assignmentManager.assignMeta();
+ ServerName currentMetaServer =
+ this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+ if (currentMetaServer != null && currentMetaServer.equals(expiredServer)) {
+ // We are expiring the server that is carrying meta already.
+ // The expiration processing will take care of reassigning meta.
+ expireIfOnline(currentMetaServer);
+ } else {
+ this.assignmentManager.assignMeta();
+ }
this.catalogTracker.waitForMeta();
// Above check waits for general meta availability but this does not
// guarantee that the transition has completed
@@ -562,6 +580,19 @@ implements HMasterInterface, HMasterRegi
return assigned;
}
+ /**
+ * Expire a server if we find it is one of the online servers set.
+ * @param sn ServerName to check.
+ * @return True if server was online and so we expired it as unreachable.
+ */
+ private boolean expireIfOnline(final ServerName sn) {
+ if (sn == null) return false;
+ if (!this.serverManager.isServerOnline(sn)) return false;
+ LOG.info("Forcing expiration of " + sn);
+ this.serverManager.expireServer(sn);
+ return true;
+ }
+
@Override
public ProtocolSignature getProtocolSignature(
String protocol, long version, int clientMethodsHashCode)