You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/04/23 04:49:20 UTC
svn commit: r1470788 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/
test/java/org/apache/hadoop/hbase/client/
Author: liyin
Date: Tue Apr 23 02:49:20 2013
New Revision: 1470788
URL: http://svn.apache.org/r1470788
Log:
[HBASE-8083] For RetriesExhausted Exception, tells whether any of the exception was thrown by RegionServer or not
Author: rshroff
Summary:
Whenever AppServer gets an errors(mostly RetriesExhaustedException),
they invalidate the cache for that user. The reason is that they are not
sure if the operation was performed on the RegionServer or not.
This change adds an API to tell whether any of the exception was thrown
by RegionServer or not. In case its not thrown by RegionServer, then
they will not invalidate the cache.
Test Plan:
mr
Will add a unit test to create different errors.
Reviewers: aaiyer, liyintang
Reviewed By: liyintang
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D761556
Task ID: 2253805
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailException.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1470788&r1=1470787&r2=1470788&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue Apr 23 02:49:20 2013
@@ -2633,7 +2633,7 @@ public class HConnectionManager {
regionFailure = failedRegionsInfo.get(regionName);
}
regionFailure.setServerName(serverName);
- regionFailure.addException(ex);
+ regionFailure.addException(ex.getCause());
}
// retry, unless it is not to be retried.
if (ex.getCause() instanceof DoNotRetryIOException) {
@@ -2664,15 +2664,6 @@ public class HConnectionManager {
if (resp != null)
result = resp.getAnswer(region);
- if (result == null || result != HConstants.MULTIPUT_SUCCESS) {
- regionName = Bytes.toStringBinary(e.getKey());
- if (!failedRegionsInfo.containsKey(regionName)) {
- regionFailure = new HRegionFailureInfo(regionName);
- failedRegionsInfo.put(regionName, regionFailure);
- }
- regionFailure.setServerName(serverName);
- regionFailure.addException(new IOException("Put failed for " + regionName));
- }
if (result == null) {
// failed
LOG.debug("Failed all for region: " +
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailException.java?rev=1470788&r1=1470787&r2=1470788&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailException.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailException.java Tue Apr 23 02:49:20 2013
@@ -40,4 +40,8 @@ public class PreemptiveFastFailException
public long getFailureCount() {
return failureCount;
}
+
+ public boolean wasOperationAttemptedByServer() {
+ return false;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java?rev=1470788&r1=1470787&r2=1470788&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java Tue Apr 23 02:49:20 2013
@@ -15,11 +15,19 @@
*/
package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.ipc.RemoteException;
+import org.mortbay.log.Log;
+import java.io.EOFException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.io.SyncFailedException;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.nio.channels.ClosedChannelException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -99,4 +107,46 @@ public class RetriesExhaustedException e
public Map<String, HRegionFailureInfo> getFailureInfo() {
return this.failureInfo;
}
+
+ /**
+ * Tells whether the operation was attempted by region server or not.
+ * @return
+ */
+ public boolean wasOperationAttemptedByServer() {
+
+ // Let's iterate through all the exceptions and if any one of them
+ // was thrown by a RegionServer, then the operation might have
+ // been tried and hence the clients should take appropriate steps
+ // to validate their cache.
+ for (HRegionFailureInfo failures : failureInfo.values()) {
+ for (Throwable e : failures.getExceptions()) {
+ if (serverOperationExecutionException(e)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Tells whether it is safe or not to consider that the current exception
+ * was thrown by the region server while executing the actual operation.
+ * @param e
+ * @return
+ */
+ private boolean serverOperationExecutionException(final Throwable e) {
+
+ // In the following scenarios it is safe to assume that the operation was not
+ // performed by the region server
+ // a) ConnectException : Client was not able to connect to the region server
+ // b) NotServingRegionException : the region is not opened on the server.
+ // c) NoServerForRegionException : there is no server present for this region.
+ //
+ // Apart from the above 3 exceptions, we are not sure if the operation was
+ // attempted on region server or not. Being conservative, we return true for
+ // all the other type of exceptions.
+ return !(e instanceof ConnectException ||
+ e instanceof NotServingRegionException ||
+ e instanceof NoServerForRegionException);
+ }
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java?rev=1470788&r1=1470787&r2=1470788&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java Tue Apr 23 02:49:20 2013
@@ -24,10 +24,24 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.InjectionEvent;
+import org.apache.hadoop.hbase.util.InjectionHandler;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -45,9 +59,9 @@ public class TestHCM {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final byte[] TABLE_NAME = Bytes.toBytes("test");
private static final byte[] FAM_NAM = Bytes.toBytes("f");
- private static final byte[] ROW = Bytes.toBytes("bbb");
+ private static final byte[] ROW = Bytes.toBytes("bbd");
- private static final int REGION_SERVERS = 10;
+ private static final int REGION_SERVERS = 3;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -77,7 +91,6 @@ public class TestHCM {
conn.deleteCachedLocation(TABLE_NAME, ROW, null);
HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
assertNull("What is this location?? " + rl, rl);
-
}
/**
@@ -197,11 +210,62 @@ public class TestHCM {
} catch (IOException e) {
} finally {
+ ServerManager.clearRSBlacklistInTest();
waitForRegionsToGetReAssigned();
}
}
/**
+ * Simulates a case where the RegionServer throws exception because
+ * a put operation failed.
+ * @throws Exception
+ */
+ @Test
+ public void testRemoteServerFailure() throws Exception {
+
+
+ HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRemoteServerFailure"), FAM_NAM);
+ table.setAutoFlush(true);
+
+ TEST_UTIL.createMultiRegions(table, FAM_NAM);
+
+ try {
+
+ for (int i = 0; i < REGION_SERVERS; i++) {
+ TEST_UTIL.getHBaseCluster().getRegionServer(i).
+ getConfiguration().set(HConstants.REGION_IMPL,
+ TestHRegion.class.getName());
+ }
+
+ TEST_UTIL.closeRegionByRow(ROW, table);
+ // Enough time for region to close and reopen with the new TestHRegionClass
+ Thread.sleep(10000);
+
+ Put put = new Put(ROW);
+ put.add(FAM_NAM, ROW, ROW);
+
+ table.put(put);
+
+ assertNull("Put should not succeed");
+ } catch (RetriesExhaustedException e) {
+ assertTrue(e.wasOperationAttemptedByServer());
+ }
+ }
+
+ static public class TestHRegion extends HRegion {
+ public TestHRegion(Path basedir, HLog log, FileSystem fs,
+ Configuration conf, HRegionInfo regionInfo, FlushRequester flushListener) {
+ super(basedir, log, fs, conf, regionInfo, flushListener);
+ }
+
+ @Override
+ public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
+ String methodName) throws IOException {
+ throw new IOException("Test induced failure");
+ }
+ }
+
+ /**
* Assign noOfRS to the dead maps list, so that no regions are assigned to
* them and then kill the current region server.
* @param table
@@ -219,10 +283,10 @@ public class TestHCM {
regionServers.get(i).getRegionServer().getHServerInfo().getHostnamePort());
}
- // abort the region server
- int srcRSIdx = TEST_UTIL.getHBaseCluster().
- getServerWith(regLoc.getRegionInfo().getRegionName());
- TEST_UTIL.getHBaseCluster().stopRegionServer(srcRSIdx);
+ HRegionInterface rsConnection =
+ TEST_UTIL.getHBaseAdmin().connection.getHRegionConnection(regLoc.getServerAddress());
+ rsConnection.closeRegion(regLoc.getRegionInfo(), false);
+
}
private void verifyFailure(HTable table, Exception e) {
@@ -236,6 +300,7 @@ public class TestHCM {
assertTrue(e instanceof RetriesExhaustedException);
RetriesExhaustedException exp = (RetriesExhaustedException)(e);
assertTrue(exp.getRegionNames() != null);
+ assertTrue(!exp.wasOperationAttemptedByServer());
LOG.info(exp.getFailureInfo().values());
}
}