You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/05/09 20:56:23 UTC
hadoop git commit: HDFS-13388. RequestHedgingProxyProvider calls
multiple configured NNs all the time. Contributed by Jinglun.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 6ce89ea2a -> c649c7656
HDFS-13388. RequestHedgingProxyProvider calls multiple configured NNs all the time. Contributed by Jinglun.
(cherry picked from commit 63803e7051dcfcd8e3d28a8b3217b7735a00b7d8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c649c765
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c649c765
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c649c765
Branch: refs/heads/branch-3.1
Commit: c649c76566cf75a9c6095797ebe012f2664e57c5
Parents: 6ce89ea
Author: Inigo Goiri <in...@apache.org>
Authored: Sun Apr 22 19:49:35 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed May 9 13:56:06 2018 -0700
----------------------------------------------------------------------
.../ha/RequestHedgingProxyProvider.java | 14 ++++
.../ha/TestRequestHedgingProxyProvider.java | 79 ++++++++++++++++++++
2 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c649c765/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
index 7b9cd64..c074bb8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
@@ -79,6 +79,20 @@ public class RequestHedgingProxyProvider<T> extends
public Object
invoke(Object proxy, final Method method, final Object[] args)
throws Throwable {
+ if (currentUsedProxy != null) {
+ try {
+ Object retVal = method.invoke(currentUsedProxy.proxy, args);
+ LOG.debug("Invocation successful on [{}]",
+ currentUsedProxy.proxyInfo);
+ return retVal;
+ } catch (InvocationTargetException ex) {
+ Exception unwrappedException = unwrapInvocationTargetException(ex);
+ logProxyException(unwrappedException, currentUsedProxy.proxyInfo);
+ LOG.trace("Unsuccessful invocation on [{}]",
+ currentUsedProxy.proxyInfo);
+ throw unwrappedException;
+ }
+ }
Map<Future<Object>, ProxyInfo<T>> proxyMap = new HashMap<>();
int numAttempts = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c649c765/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 8d6b02d..8913f1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -43,10 +43,15 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.junit.Assert.assertEquals;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.mock;
import com.google.common.collect.Lists;
@@ -100,6 +105,80 @@ public class TestRequestHedgingProxyProvider {
}
@Test
+ public void testRequestNNAfterOneSuccess() throws Exception {
+ final AtomicInteger goodCount = new AtomicInteger(0);
+ final AtomicInteger badCount = new AtomicInteger(0);
+ final ClientProtocol goodMock = mock(ClientProtocol.class);
+ when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
+ @Override
+ public long[] answer(InvocationOnMock invocation) throws Throwable {
+ goodCount.incrementAndGet();
+ Thread.sleep(1000);
+ return new long[]{1};
+ }
+ });
+ final ClientProtocol badMock = mock(ClientProtocol.class);
+ when(badMock.getStats()).thenAnswer(new Answer<long[]>() {
+ @Override
+ public long[] answer(InvocationOnMock invocation) throws Throwable {
+ badCount.incrementAndGet();
+ throw new IOException("Bad mock !!");
+ }
+ });
+
+ RequestHedgingProxyProvider<ClientProtocol> provider =
+ new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class,
+ createFactory(badMock, goodMock));
+ ClientProtocol proxy = provider.getProxy().proxy;
+ proxy.getStats();
+ assertEquals(1, goodCount.get());
+ assertEquals(1, badCount.get());
+ // We will only use the successful proxy after a successful invocation.
+ proxy.getStats();
+ assertEquals(2, goodCount.get());
+ assertEquals(1, badCount.get());
+ }
+
+ @Test
+ public void testExceptionInfo() throws Exception {
+ final ClientProtocol goodMock = mock(ClientProtocol.class);
+ when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
+ private boolean first = true;
+ @Override
+ public long[] answer(InvocationOnMock invocation)
+ throws Throwable {
+ if (first) {
+ Thread.sleep(1000);
+ first = false;
+ return new long[] {1};
+ } else {
+ throw new IOException("Expected Exception Info");
+ }
+ }
+ });
+ final ClientProtocol badMock = mock(ClientProtocol.class);
+ when(badMock.getStats()).thenAnswer(new Answer<long[]>() {
+ @Override
+ public long[] answer(InvocationOnMock invocation)
+ throws Throwable {
+ throw new IOException("Bad Mock! This is Standby!");
+ }
+ });
+
+ RequestHedgingProxyProvider<ClientProtocol> provider =
+ new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class,
+ createFactory(badMock, goodMock));
+ ClientProtocol proxy = provider.getProxy().proxy;
+ proxy.getStats();
+ // Test getting the exception when the successful proxy encounters one.
+ try {
+ proxy.getStats();
+ } catch (IOException e) {
+ assertExceptionContains("Expected Exception Info", e);
+ }
+ }
+
+ @Test
public void testHedgingWhenOneIsSlow() throws Exception {
final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class);
Mockito.when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org