You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2015/11/15 21:03:53 UTC
[29/50] [abbrv] hadoop git commit: HDFS-9364. Unnecessary DNS
resolution attempts when creating NameNodeProxies. Contributed by Xiao Chen.
HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. Contributed by Xiao Chen.
Change-Id: I9e42f724f27924cf73891425a832de37ce014a1e
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73b94d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73b94d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73b94d78
Branch: refs/heads/yarn-2877
Commit: 73b94d789969354bb9a6872d99976763ca8470d7
Parents: 0eb9c60
Author: Zhe Zhang <zh...@apache.org>
Authored: Tue Nov 10 09:55:29 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Nov 10 09:55:29 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 +++++++++++-
.../hadoop/hdfs/NameNodeProxiesClient.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/TestDFSClientFailover.java | 35 ++++++++++++++++++++
4 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index c14d5d3..62c5d81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -615,7 +615,7 @@ public class DFSUtilClient {
public static InetSocketAddress getNNAddress(Configuration conf) {
URI filesystemURI = FileSystem.getDefaultUri(conf);
- return getNNAddress(filesystemURI);
+ return getNNAddressCheckLogical(conf, filesystemURI);
}
/**
@@ -638,6 +638,26 @@ public class DFSUtilClient {
return getNNAddress(authority);
}
+ /**
+ * Get the NN address from the URI. If the uri is logical, default address is
+ * returned. Otherwise return the DNS-resolved address of the URI.
+ *
+ * @param conf configuration
+ * @param filesystemURI URI of the file system
+ * @return address of file system
+ */
+ public static InetSocketAddress getNNAddressCheckLogical(Configuration conf,
+ URI filesystemURI) {
+ InetSocketAddress retAddr;
+ if (HAUtilClient.isLogicalUri(conf, filesystemURI)) {
+ retAddr = InetSocketAddress.createUnresolved(filesystemURI.getAuthority(),
+ HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
+ } else {
+ retAddr = getNNAddress(filesystemURI);
+ }
+ return retAddr;
+ }
+
public static URI getNNUri(InetSocketAddress namenode) {
int port = namenode.getPort();
String portString =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
index 39b188f..5ca7030 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
@@ -320,7 +320,7 @@ public class NameNodeProxiesClient {
DFSUtilClient.getNNAddress(nameNodeUri));
}
return new ProxyAndInfo<>(proxy, dtService,
- DFSUtilClient.getNNAddress(nameNodeUri));
+ DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri));
}
public static ClientProtocol createNonHAProxyWithClientProtocol(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8625ecf..336db03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2285,6 +2285,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9401. Fix findbugs warnings in BlockRecoveryWorker.
(Brahma Reddy Battula via waltersu4549)
+ HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies.
+ (Xiao Chen via zhz)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
index b098711..60ffe30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.net.ConnectTimeoutException;
@@ -291,6 +292,40 @@ public class TestDFSClientFailover {
Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
}
+ /**
+ * Test that creating proxy doesn't ever try to DNS-resolve the logical URI.
+ * Regression test for HDFS-9364.
+ */
+ @Test(timeout=60000)
+ public void testCreateProxyDoesntDnsResolveLogicalURI() throws IOException {
+ final NameService spyNS = spyOnNameService();
+ final Configuration conf = new HdfsConfiguration();
+ final String service = "nameservice1";
+ final String namenode = "namenode113";
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, service);
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://" + service);
+ conf.set(
+ HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + service,
+ "org.apache.hadoop.hdfs.server.namenode.ha."
+ + "ConfiguredFailoverProxyProvider");
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + service,
+ namenode);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + service + "."
+ + namenode, "localhost:8020");
+
+ // call createProxy implicitly and explicitly
+ Path p = new Path("/");
+ p.getFileSystem(conf);
+ NameNodeProxiesClient.createProxyWithClientProtocol(conf,
+ FileSystem.getDefaultUri(conf), null);
+ NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
+ NamenodeProtocol.class, null);
+
+ // Ensure that the logical hostname was never resolved.
+ Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(
+ Mockito.eq(service));
+ }
+
/** Dummy implementation of plain FailoverProxyProvider */
public static class DummyLegacyFailoverProxyProvider<T>
implements FailoverProxyProvider<T> {