You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2019/12/11 17:48:40 UTC
[hadoop] branch branch-3.1 updated: HDFS-15032. Properly handle
InvocationTargetExceptions in the proxy created by ProxyCombiner. This
fixes a bug encountered by the HDFS balancer when used with Observer Nodes.
Contributed by Erik Krogen.
This is an automated email from the ASF dual-hosted git repository.
xkrogen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b06965a HDFS-15032. Properly handle InvocationTargetExceptions in the proxy created by ProxyCombiner. This fixes a bug encountered by the HDFS balancer when used with Observer Nodes. Contributed by Erik Krogen.
b06965a is described below
commit b06965ae32c4be2e6923a2f057290fbdbda5f38a
Author: Erik Krogen <xk...@apache.org>
AuthorDate: Wed Dec 4 10:54:03 2019 -0800
HDFS-15032. Properly handle InvocationTargetExceptions in the proxy created by ProxyCombiner. This fixes a bug encountered by the HDFS balancer when used with Observer Nodes. Contributed by Erik Krogen.
(cherry picked from c174d50b30abc08a4642614fb35165e79792608b)
(cherry picked from 3402c87353c18730cb659ac1d8b43b4e21c634f6)
---
.../java/org/apache/hadoop/ipc/ProxyCombiner.java | 18 ++++++++++++--
.../balancer/TestBalancerWithHANameNodes.java | 28 ++++++++++++++++++----
2 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
index fbafabc..99eb487 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
@@ -17,9 +17,11 @@
*/
package org.apache.hadoop.ipc;
+import com.google.common.base.Joiner;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
@@ -74,7 +76,8 @@ public final class ProxyCombiner {
+ combinedProxyInterface + " do not cover method " + m);
}
- InvocationHandler handler = new CombinedProxyInvocationHandler(proxies);
+ InvocationHandler handler =
+ new CombinedProxyInvocationHandler(combinedProxyInterface, proxies);
return (T) Proxy.newProxyInstance(combinedProxyInterface.getClassLoader(),
new Class[] {combinedProxyInterface}, handler);
}
@@ -82,9 +85,12 @@ public final class ProxyCombiner {
private static final class CombinedProxyInvocationHandler
implements RpcInvocationHandler {
+ private final Class<?> proxyInterface;
private final Object[] proxies;
- private CombinedProxyInvocationHandler(Object[] proxies) {
+ private CombinedProxyInvocationHandler(Class<?> proxyInterface,
+ Object[] proxies) {
+ this.proxyInterface = proxyInterface;
this.proxies = proxies;
}
@@ -97,6 +103,8 @@ public final class ProxyCombiner {
return method.invoke(underlyingProxy, args);
} catch (IllegalAccessException|IllegalArgumentException e) {
lastException = e;
+ } catch (InvocationTargetException ite) {
+ throw ite.getCause();
}
}
// This shouldn't happen since the method coverage was verified in build()
@@ -117,6 +125,12 @@ public final class ProxyCombiner {
}
@Override
+ public String toString() {
+ return "CombinedProxy[" + proxyInterface.getSimpleName() + "]["
+ + Joiner.on(",").join(proxies) + "]";
+ }
+
+ @Override
public void close() throws IOException {
MultipleIOException.Builder exceptionBuilder =
new MultipleIOException.Builder();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index fd9d037..f86a522 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -125,10 +125,25 @@ public class TestBalancerWithHANameNodes {
/**
* Test Balancer with ObserverNodes.
*/
- @Test(timeout = 60000)
+ @Test(timeout = 120000)
public void testBalancerWithObserver() throws Exception {
+ testBalancerWithObserver(false);
+ }
+
+ /**
+ * Test Balancer with ObserverNodes when one has failed.
+ */
+ @Test(timeout = 180000)
+ public void testBalancerWithObserverWithFailedNode() throws Exception {
+ testBalancerWithObserver(true);
+ }
+
+ private void testBalancerWithObserver(boolean withObserverFailure)
+ throws Exception {
final Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
+ // Avoid the same FS being reused between tests
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
MiniQJMHACluster qjmhaCluster = null;
try {
@@ -142,6 +157,10 @@ public class TestBalancerWithHANameNodes {
namesystemSpies.add(
NameNodeAdapter.spyOnNamesystem(cluster.getNameNode(i)));
}
+ if (withObserverFailure) {
+ // First observer NN is at index 2
+ cluster.shutdownNameNode(2);
+ }
DistributedFileSystem dfs = HATestUtil.configureObserverReadFs(
cluster, conf, ObserverReadProxyProvider.class, true);
@@ -149,9 +168,10 @@ public class TestBalancerWithHANameNodes {
doTest(conf);
for (int i = 0; i < cluster.getNumNameNodes(); i++) {
- // First observer node is at idx 2 so it should get both getBlocks calls
- // all other NameNodes should see 0 getBlocks calls
- int expectedCount = (i == 2) ? 2 : 0;
+ // First observer node is at idx 2, or 3 if 2 has been shut down
+ // It should get both getBlocks calls, all other NNs should see 0 calls
+ int expectedObserverIdx = withObserverFailure ? 3 : 2;
+ int expectedCount = (i == expectedObserverIdx) ? 2 : 0;
verify(namesystemSpies.get(i), times(expectedCount))
.getBlocks(any(), anyLong(), anyLong());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org