You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:07:45 UTC
[09/50] [abbrv] git commit: [master] Clear rootRegionLocation if the
root is not served there
[master] Clear rootRegionLocation if the root is not served there
Summary: Clear `TableServers.rootRegionLocation` when receiving `NotServingRegionException`
Test Plan: `TestGetRegionLocation`
Reviewers: elliott, fan, manukranthk
Reviewed By: fan, manukranthk
Subscribers: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1367998
Tasks: 4448200
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42760 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df1f91fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df1f91fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df1f91fa
Branch: refs/heads/0.89-fb
Commit: df1f91fad67a85315a80ff39beca640ea33ff4f4
Parents: 91eea3f
Author: daviddeng <da...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 9 19:02:34 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 2 +-
.../apache/hadoop/hbase/HTableDescriptor.java | 4 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 4 +-
.../hadoop/hbase/client/HTableMultiplexer.java | 24 +++----
.../hadoop/hbase/client/TableServers.java | 20 +++---
.../hadoop/hbase/HBaseTestingUtility.java | 31 +++++++++
.../regionserver/TestGetRegionLocation.java | 69 +++++++++++++++++++-
7 files changed, 126 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index dee47e0..9b30994 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -518,7 +518,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return this.tableDesc.isRootRegion();
}
- /** @return true if this is the meta table */
+ /** @return true if this is the META table */
public boolean isMetaTable() {
return this.tableDesc.isMetaTable();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 6f9b81a..5ddddfb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -35,7 +35,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
-import com.google.common.collect.ImmutableSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +46,7 @@ import org.apache.hadoop.io.WritableComparable;
import com.facebook.swift.codec.ThriftConstructor;
import com.facebook.swift.codec.ThriftField;
import com.facebook.swift.codec.ThriftStruct;
+import com.google.common.collect.ImmutableSet;
/**
* HTableDescriptor contains the name of an HTable, and its
@@ -324,7 +324,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
values.put(IS_META_KEY, isMeta? TRUE: FALSE);
}
- /** @return true if table is the meta table */
+ /** @return true if table is the META table */
public boolean isMetaTable() {
return isMetaRegion() && !isRootRegion();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ffeaf81..c16889b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -91,7 +91,7 @@ public class HBaseAdmin {
// use 30s pause time between retries instead of the small default
// pause time. Shouldn't the RPC read timeout be also increased?
this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, 30 * 1000);
- this.numRetries = conf.getInt("hbase.client.retries.number", 5);
+ this.numRetries = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
this.master = connection.getMaster();
}
@@ -1441,7 +1441,7 @@ public class HBaseAdmin {
public static void checkHBaseAvailable(Configuration conf)
throws MasterNotRunningException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
- copyOfConf.setInt("hbase.client.retries.number", 1);
+ copyOfConf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 1);
new HBaseAdmin(copyOfConf);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
index a3360c0..dfee8c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
@@ -57,13 +57,13 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* Each put will be sharded into different buffer queues based on its destination region server.
* So each region server buffer queue will only have the puts which share the same destination.
* And each queue will have a flush worker thread to flush the puts request to the region server.
- * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that
+ * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that
* particular queue.
- *
+ *
* Also all the puts will be retried as a configuration number before dropping.
* And the HTableMultiplexer can report the number of buffered requests and the number of the
* failed (dropped) requests in total or on per region server basis.
- *
+ *
* This class is thread safe.
*/
public class HTableMultiplexer {
@@ -87,11 +87,11 @@ public class HTableMultiplexer {
private final long frequency;
//initial number of threads in the pool
public static final int INITIAL_NUM_THREADS = 10;
-
+
/**
- *
+ *
* @param conf The HBaseConfiguration
- * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops
+ * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops
* for each region server before dropping the request.
*/
public HTableMultiplexer(Configuration conf, int perRegionServerBufferQueueSize) {
@@ -102,7 +102,7 @@ public class HTableMultiplexer {
this.serverToFlushWorkerMap = new ConcurrentHashMap<HServerAddress, HTableFlushWorker>();
this.tableNameToHTableMap = new ConcurrentSkipListMap<byte[], HTable>(
Bytes.BYTES_COMPARATOR);
- this.retryNum = conf.getInt("hbase.client.retries.number", 10);
+ this.retryNum = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 10);
this.retriedInQueueMax = conf.getInt("hbase.client.retried.inQueue", 10000);
this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize;
this.frequency = conf.getLong("hbase.htablemultiplexer.flush.frequency.ms",
@@ -127,7 +127,7 @@ public class HTableMultiplexer {
}
/**
- * The puts request will be buffered by their corresponding buffer queue.
+ * The puts request will be buffered by their corresponding buffer queue.
* Return the list of puts which could not be queued.
* @param table
* @param put
@@ -138,13 +138,13 @@ public class HTableMultiplexer {
HBaseRPCOptions options) throws IOException {
if (puts == null)
return null;
-
+
List <Put> failedPuts = null;
boolean result;
for (Put put : puts) {
result = put(table, put, this.retryNum, options);
if (result == false) {
-
+
// Create the failed puts list if necessary
if (failedPuts == null) {
failedPuts = new ArrayList<Put>();
@@ -183,7 +183,7 @@ public class HTableMultiplexer {
LinkedBlockingQueue<PutStatus> queue = getBufferedQueue(addr);
// Generate a MultiPutStatus obj and offer it into the queue
PutStatus s = new PutStatus(loc.getRegionInfo(), put, retry, options);
-
+
return queue.offer(s);
}
} catch (Exception e) {
@@ -480,7 +480,7 @@ public class HTableMultiplexer {
return this.tableNameToHTableMap.size();
}
}
-
+
private static class PutStatus {
private final HRegionInfo regionInfo;
private final Put put;
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index bc38cd6..bb671fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -51,11 +51,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.ListeningExecutorService;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -97,8 +94,11 @@ import org.apache.hadoop.ipc.RemoteException;
import org.apache.thrift.transport.TTransportException;
import com.google.common.base.Preconditions;
-
-import javax.annotation.Nullable;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningExecutorService;
/* Encapsulates finding the servers for an HBase instance */
public class TableServers implements ServerConnection {
@@ -1079,8 +1079,6 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
parentTable, row);
metaCache.add(tableName, location);
}
- } catch (Throwable t) {
- throw t;
} finally {
HBaseThriftRPC.isMeta.get().pop();
}
@@ -1094,6 +1092,10 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
// already processed this. Don't process this again.
throw e;
} catch (IOException e) {
+ if (e instanceof NotServingRegionException) {
+ this.rootRegionLocation = null;
+ }
+
if (e instanceof RemoteException) {
e = RemoteExceptionHandler
.decodeRemoteException((RemoteException) e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 101e552..10762fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1561,6 +1561,37 @@ public class HBaseTestingUtility {
return list;
}
+ /**
+ * @return the only HRegion for ROOT table. Returns a null if not found.
+ * It loops over all regions to find it, could be slow.
+ */
+ public HRegion getRootRegion() {
+ for (HRegionServer rs : getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ if (region.getRegionInfo().isRootRegion()) {
+ return region;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * @return one of HRegion for META table. Returns a null if not found.
+ * It loops over all regions to find it, could be slow.
+ */
+ public HRegion getMetaRegion() {
+ for (HRegionServer rs : getOnlineRegionServers()) {
+ for (HRegion region : rs.getOnlineRegions()) {
+ if (region.getRegionInfo().isMetaTable()) {
+ return region;
+ }
+ }
+ }
+
+ return null;
+ }
/**
* Wait until <code>countOfRegion</code> in .META. have a non-empty
http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
index efb24b4..de391c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
@@ -30,13 +30,17 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.TableServers;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -48,7 +52,7 @@ public class TestGetRegionLocation {
private static final Log LOG = LogFactory.getLog(TestGetRegionLocation.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static Configuration CONF = TEST_UTIL.getConfiguration();
- private final static byte[] TABLE = Bytes.toBytes("table");
+ private final static StringBytes TABLE = new StringBytes("table");
private final static byte[] FAMILY = Bytes.toBytes("family");
private final static byte[][] FAMILIES = { FAMILY };
private final static byte[] START_KEY = Bytes.toBytes("aaa");
@@ -58,10 +62,13 @@ public class TestGetRegionLocation {
@Before
public void setUp() throws IOException, InterruptedException {
+ TEST_UTIL.getConfiguration().setInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
+
// Use assignment plan so that regions are not moved unexpectedly.
TEST_UTIL.useAssignmentLoadBalancer();
TEST_UTIL.startMiniCluster(NUM_SLAVES);
- TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS);
+ TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS)
+ .close();
}
@After
@@ -69,6 +76,64 @@ public class TestGetRegionLocation {
TEST_UTIL.shutdownMiniCluster();
}
+ @Test(timeout = 150000)
+ public void testMetaRootRegionRelocation() throws Exception {
+ Assert.assertTrue("We should have more than one region servers",
+ NUM_SLAVES >= 3);
+
+ try (TableServers connection = new TableServers(CONF)) {
+ HRegionLocation location =
+ connection.getRegionLocation(TABLE, HConstants.EMPTY_START_ROW, true);
+ Assert.assertNotNull("Cannot get location", location);
+
+ HRegion rootRegion = TEST_UTIL.getRootRegion();
+ Assert.assertNotNull("Cannot find ROOT region", rootRegion);
+ HRegionServer rootServer =
+ TEST_UTIL.getRSWithRegion(rootRegion.getRegionName());
+
+ HRegion metaRegion = TEST_UTIL.getMetaRegion();
+ Assert.assertNotNull("Cannot find META region", metaRegion);
+ HRegionServer metaServer =
+ TEST_UTIL.getRSWithRegion(metaRegion.getRegionName());
+
+ TEST_UTIL.getMiniHBaseCluster().getMaster()
+ .addServerToBlacklist(rootServer.getServerInfo().getHostnamePort());
+ TEST_UTIL.getMiniHBaseCluster().getMaster()
+ .addServerToBlacklist(metaServer.getServerInfo().getHostnamePort());
+
+ rootServer.closeRegion(rootRegion.getRegionInfo(), true);
+
+ metaServer.closeRegion(metaRegion.getRegionInfo(), true);
+
+ try {
+ rootServer.getRegion(rootRegion.getRegionName());
+ Assert.fail("Should throw NotServingRegionException");
+ } catch (NotServingRegionException e) {
+ // good
+ }
+ try {
+ metaServer.getRegion(metaRegion.getRegionName());
+ Assert.fail("Should throw NotServingRegionException");
+ } catch (NotServingRegionException e) {
+ // good
+ }
+
+ try {
+ connection.listTables();
+ } catch (RetriesExhaustedException e) {
+ e.printStackTrace();
+ Assert.fail("listTable failed: " + e);
+ }
+
+ try {
+ connection.listTables();
+ } catch (NotServingRegionException e) {
+ e.printStackTrace();
+ Assert.fail("listTable failed: " + e);
+ }
+ }
+ }
+
/**
* This test tests the getRegionLocation() call in the
* (Thrift)HRegionInterface API.